HMVMXR0.cpp@ 81239

Last change on this file since 81239 was 81239, checked in by vboxsync, 5 years ago
VMM/HMVMXR0: Nit.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 708.0 KB

Line
1	/* $Id: HMVMXR0.cpp 81239 2019-10-14 07:11:33Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#include "HMInternal.h"
39	#include <VBox/vmm/vmcc.h>
40	#include <VBox/vmm/hmvmxinline.h>
41	#include "HMVMXR0.h"
42	#include "dtrace/VBoxVMM.h"
43
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
47	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/**
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
83	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
84
85	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
86	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
87	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
88	\| HMVMX_READ_EXIT_INSTR_LEN \
89	\| HMVMX_READ_IDT_VECTORING_INFO \
90	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
91
92	/** Assert that all the given fields have been read from the VMCS. */
93	#ifdef VBOX_STRICT
94	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
95	do { \
96	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
97	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); \
98	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
99	} while (0)
100	#else
101	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
102	#endif
103
104	/**
105	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
106	* guest using hardware-assisted VMX.
107	*
108	* This excludes state like GPRs (other than RSP) which are always are
109	* swapped and restored across the world-switch and also registers like EFER,
110	* MSR which cannot be modified by the guest without causing a VM-exit.
111	*/
112	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
113	\| CPUMCTX_EXTRN_RFLAGS \
114	\| CPUMCTX_EXTRN_RSP \
115	\| CPUMCTX_EXTRN_SREG_MASK \
116	\| CPUMCTX_EXTRN_TABLE_MASK \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_TSC_AUX \
121	\| CPUMCTX_EXTRN_OTHER_MSRS \
122	\| CPUMCTX_EXTRN_CR0 \
123	\| CPUMCTX_EXTRN_CR3 \
124	\| CPUMCTX_EXTRN_CR4 \
125	\| CPUMCTX_EXTRN_DR7 \
126	\| CPUMCTX_EXTRN_HWVIRT \
127	\| CPUMCTX_EXTRN_HM_VMX_MASK)
128
129	/**
130	* Exception bitmap mask for real-mode guests (real-on-v86).
131	*
132	* We need to intercept all exceptions manually except:
133	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
134	* due to bugs in Intel CPUs.
135	* - \#PF need not be intercepted even in real-mode if we have nested paging
136	* support.
137	*/
138	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
139	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
140	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
141	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
142	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
143	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
144	\| RT_BIT(X86_XCPT_XF))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158	/** Assert that preemption is disabled or covered by thread-context hooks. */
159	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
160	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
161
162	/** Assert that we haven't migrated CPUs when thread-context hooks are not
163	* used. */
164	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
165	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
166	("Illegal migration! Entered on CPU %u Current %u\n", \
167	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
168
169	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
170	* context. */
171	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
172	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
173	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
174
175	/** Log the VM-exit reason with an easily visible marker to identify it in a
176	* potential sea of logging data. */
177	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
178	do { \
179	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
180	HMGetVmxExitName(a_uExitReason))); \
181	} while (0) \
182
183
184	/*********************************************************************************************************************************
185	* Structures and Typedefs *
186	*********************************************************************************************************************************/
187	/**
188	* VMX per-VCPU transient state.
189	*
190	* A state structure for holding miscellaneous information across
191	* VMX non-root operation and restored after the transition.
192	*/
193	typedef struct VMXTRANSIENT
194	{
195	/** The host's rflags/eflags. */
196	RTCCUINTREG fEFlags;
197
198	/** The guest's TPR value used for TPR shadowing. */
199	uint8_t u8GuestTpr;
200	/** Alignment. */
201	uint8_t abAlignment0[7];
202
203	/** The basic VM-exit reason. */
204	uint16_t uExitReason;
205	/** Alignment. */
206	uint16_t u16Alignment0;
207	/** The VM-exit interruption error code. */
208	uint32_t uExitIntErrorCode;
209	/** The VM-exit exit code qualification. */
210	uint64_t uExitQual;
211	/** The Guest-linear address. */
212	uint64_t uGuestLinearAddr;
213	/** The Guest-physical address. */
214	uint64_t uGuestPhysicalAddr;
215	/** The Guest pending-debug exceptions. */
216	uint64_t uGuestPendingDbgXcpts;
217
218	/** The VM-exit interruption-information field. */
219	uint32_t uExitIntInfo;
220	/** The VM-exit instruction-length field. */
221	uint32_t cbExitInstr;
222	/** The VM-exit instruction-information field. */
223	VMXEXITINSTRINFO ExitInstrInfo;
224	/** Whether the VM-entry failed or not. */
225	bool fVMEntryFailed;
226	/** Whether we are currently executing a nested-guest. */
227	bool fIsNestedGuest;
228	/** Alignment. */
229	uint8_t abAlignment1[2];
230
231	/** The VM-entry interruption-information field. */
232	uint32_t uEntryIntInfo;
233	/** The VM-entry exception error code field. */
234	uint32_t uEntryXcptErrorCode;
235	/** The VM-entry instruction length field. */
236	uint32_t cbEntryInstr;
237
238	/** IDT-vectoring information field. */
239	uint32_t uIdtVectoringInfo;
240	/** IDT-vectoring error code. */
241	uint32_t uIdtVectoringErrorCode;
242
243	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
244	uint32_t fVmcsFieldsRead;
245
246	/** Whether the guest debug state was active at the time of VM-exit. */
247	bool fWasGuestDebugStateActive;
248	/** Whether the hyper debug state was active at the time of VM-exit. */
249	bool fWasHyperDebugStateActive;
250	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
251	bool fUpdatedTscOffsettingAndPreemptTimer;
252	/** Whether the VM-exit was caused by a page-fault during delivery of a
253	* contributory exception or a page-fault. */
254	bool fVectoringDoublePF;
255	/** Whether the VM-exit was caused by a page-fault during delivery of an
256	* external interrupt or NMI. */
257	bool fVectoringPF;
258	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
259	* area after VM-exit. */
260	bool fRemoveTscAuxMsr;
261	bool afAlignment0[2];
262
263	/** The VMCS info. object. */
264	PVMXVMCSINFO pVmcsInfo;
265	} VMXTRANSIENT;
266	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
267	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
268	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
269	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
271	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
272	/** Pointer to VMX transient state. */
273	typedef VMXTRANSIENT *PVMXTRANSIENT;
274	/** Pointer to a const VMX transient state. */
275	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
276
277	/**
278	* Memory operand read or write access.
279	*/
280	typedef enum VMXMEMACCESS
281	{
282	VMXMEMACCESS_READ = 0,
283	VMXMEMACCESS_WRITE = 1
284	} VMXMEMACCESS;
285
286	/**
287	* VMX VM-exit handler.
288	*
289	* @returns Strict VBox status code (i.e. informational status codes too).
290	* @param pVCpu The cross context virtual CPU structure.
291	* @param pVmxTransient The VMX-transient structure.
292	*/
293	#ifndef HMVMX_USE_FUNCTION_TABLE
294	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
295	#else
296	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
297	/** Pointer to VM-exit handler. */
298	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
299	#endif
300
301	/**
302	* VMX VM-exit handler, non-strict status code.
303	*
304	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
305	*
306	* @returns VBox status code, no informational status code returned.
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*
310	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
311	* use of that status code will be replaced with VINF_EM_SOMETHING
312	* later when switching over to IEM.
313	*/
314	#ifndef HMVMX_USE_FUNCTION_TABLE
315	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
316	#else
317	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
318	#endif
319
320
321	/*********************************************************************************************************************************
322	* Internal Functions *
323	*********************************************************************************************************************************/
324	#ifndef HMVMX_USE_FUNCTION_TABLE
325	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
326	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
327	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
328	#else
329	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
330	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
331	#endif
332	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
333	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
334	#endif
335
336	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
337
338	/** @name VM-exit handler prototypes.
339	* @{
340	*/
341	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
342	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
343	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
344	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
345	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
346	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
347	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
348	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
349	static FNVMXEXITHANDLER hmR0VmxExitHlt;
350	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
351	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
352	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
353	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
354	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
355	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
356	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
357	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
358	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
359	static FNVMXEXITHANDLER hmR0VmxExitVmread;
360	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
361	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
362	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
363	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
364	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
365	#endif
366	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
367	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
368	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
369	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
370	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
371	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
372	static FNVMXEXITHANDLER hmR0VmxExitMwait;
373	static FNVMXEXITHANDLER hmR0VmxExitMtf;
374	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
375	static FNVMXEXITHANDLER hmR0VmxExitPause;
376	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
377	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
378	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
379	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
380	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
381	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
382	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
383	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
384	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
385	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
386	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
388	/** @} */
389
390	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
391	/** @name Nested-guest VM-exit handler prototypes.
392	* @{
393	*/
394	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
395	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
398	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
399	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
400	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
401	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
402	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
403	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
404	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
405	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
406	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
407	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
408	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
409	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
410	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
411	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
412	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
414	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
415	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
416	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
417	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
418	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
419	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
421	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
422	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
423	/** @} */
424	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
425
426
427	/*********************************************************************************************************************************
428	* Global Variables *
429	*********************************************************************************************************************************/
430	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
431	/**
432	* Array of all VMCS fields.
433	* Any fields added to the VT-x spec. should be added here.
434	*
435	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
436	* of nested-guests.
437	*/
438	static const uint32_t g_aVmcsFields[] =
439	{
440	/* 16-bit control fields. */
441	VMX_VMCS16_VPID,
442	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
443	VMX_VMCS16_EPTP_INDEX,
444
445	/* 16-bit guest-state fields. */
446	VMX_VMCS16_GUEST_ES_SEL,
447	VMX_VMCS16_GUEST_CS_SEL,
448	VMX_VMCS16_GUEST_SS_SEL,
449	VMX_VMCS16_GUEST_DS_SEL,
450	VMX_VMCS16_GUEST_FS_SEL,
451	VMX_VMCS16_GUEST_GS_SEL,
452	VMX_VMCS16_GUEST_LDTR_SEL,
453	VMX_VMCS16_GUEST_TR_SEL,
454	VMX_VMCS16_GUEST_INTR_STATUS,
455	VMX_VMCS16_GUEST_PML_INDEX,
456
457	/* 16-bits host-state fields. */
458	VMX_VMCS16_HOST_ES_SEL,
459	VMX_VMCS16_HOST_CS_SEL,
460	VMX_VMCS16_HOST_SS_SEL,
461	VMX_VMCS16_HOST_DS_SEL,
462	VMX_VMCS16_HOST_FS_SEL,
463	VMX_VMCS16_HOST_GS_SEL,
464	VMX_VMCS16_HOST_TR_SEL,
465
466	/* 64-bit control fields. */
467	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
468	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
469	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
470	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
471	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
472	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
473	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
474	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
475	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
476	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
477	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
478	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
479	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
480	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
481	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
482	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
483	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
484	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
485	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
486	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
487	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
488	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
489	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
490	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
491	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
492	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
493	VMX_VMCS64_CTRL_EPTP_FULL,
494	VMX_VMCS64_CTRL_EPTP_HIGH,
495	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
496	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
497	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
498	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
499	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
500	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
501	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
502	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
503	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
504	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
505	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
506	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
507	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
508	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
509	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
510	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
511	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
512	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
513	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
514	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
515	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
516	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
517
518	/* 64-bit read-only data fields. */
519	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
520	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
521
522	/* 64-bit guest-state fields. */
523	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
524	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
525	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
526	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
527	VMX_VMCS64_GUEST_PAT_FULL,
528	VMX_VMCS64_GUEST_PAT_HIGH,
529	VMX_VMCS64_GUEST_EFER_FULL,
530	VMX_VMCS64_GUEST_EFER_HIGH,
531	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
532	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
533	VMX_VMCS64_GUEST_PDPTE0_FULL,
534	VMX_VMCS64_GUEST_PDPTE0_HIGH,
535	VMX_VMCS64_GUEST_PDPTE1_FULL,
536	VMX_VMCS64_GUEST_PDPTE1_HIGH,
537	VMX_VMCS64_GUEST_PDPTE2_FULL,
538	VMX_VMCS64_GUEST_PDPTE2_HIGH,
539	VMX_VMCS64_GUEST_PDPTE3_FULL,
540	VMX_VMCS64_GUEST_PDPTE3_HIGH,
541	VMX_VMCS64_GUEST_BNDCFGS_FULL,
542	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
543
544	/* 64-bit host-state fields. */
545	VMX_VMCS64_HOST_PAT_FULL,
546	VMX_VMCS64_HOST_PAT_HIGH,
547	VMX_VMCS64_HOST_EFER_FULL,
548	VMX_VMCS64_HOST_EFER_HIGH,
549	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
550	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
551
552	/* 32-bit control fields. */
553	VMX_VMCS32_CTRL_PIN_EXEC,
554	VMX_VMCS32_CTRL_PROC_EXEC,
555	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
556	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
557	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
558	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
559	VMX_VMCS32_CTRL_EXIT,
560	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
561	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
562	VMX_VMCS32_CTRL_ENTRY,
563	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
564	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
565	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
566	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
567	VMX_VMCS32_CTRL_TPR_THRESHOLD,
568	VMX_VMCS32_CTRL_PROC_EXEC2,
569	VMX_VMCS32_CTRL_PLE_GAP,
570	VMX_VMCS32_CTRL_PLE_WINDOW,
571
572	/* 32-bits read-only fields. */
573	VMX_VMCS32_RO_VM_INSTR_ERROR,
574	VMX_VMCS32_RO_EXIT_REASON,
575	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
576	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
577	VMX_VMCS32_RO_IDT_VECTORING_INFO,
578	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
579	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
580	VMX_VMCS32_RO_EXIT_INSTR_INFO,
581
582	/* 32-bit guest-state fields. */
583	VMX_VMCS32_GUEST_ES_LIMIT,
584	VMX_VMCS32_GUEST_CS_LIMIT,
585	VMX_VMCS32_GUEST_SS_LIMIT,
586	VMX_VMCS32_GUEST_DS_LIMIT,
587	VMX_VMCS32_GUEST_FS_LIMIT,
588	VMX_VMCS32_GUEST_GS_LIMIT,
589	VMX_VMCS32_GUEST_LDTR_LIMIT,
590	VMX_VMCS32_GUEST_TR_LIMIT,
591	VMX_VMCS32_GUEST_GDTR_LIMIT,
592	VMX_VMCS32_GUEST_IDTR_LIMIT,
593	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
594	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
595	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
596	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
597	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
598	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
599	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
600	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_INT_STATE,
602	VMX_VMCS32_GUEST_ACTIVITY_STATE,
603	VMX_VMCS32_GUEST_SMBASE,
604	VMX_VMCS32_GUEST_SYSENTER_CS,
605	VMX_VMCS32_PREEMPT_TIMER_VALUE,
606
607	/* 32-bit host-state fields. */
608	VMX_VMCS32_HOST_SYSENTER_CS,
609
610	/* Natural-width control fields. */
611	VMX_VMCS_CTRL_CR0_MASK,
612	VMX_VMCS_CTRL_CR4_MASK,
613	VMX_VMCS_CTRL_CR0_READ_SHADOW,
614	VMX_VMCS_CTRL_CR4_READ_SHADOW,
615	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
616	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
617	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
618	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
619
620	/* Natural-width read-only data fields. */
621	VMX_VMCS_RO_EXIT_QUALIFICATION,
622	VMX_VMCS_RO_IO_RCX,
623	VMX_VMCS_RO_IO_RSI,
624	VMX_VMCS_RO_IO_RDI,
625	VMX_VMCS_RO_IO_RIP,
626	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
627
628	/* Natural-width guest-state field */
629	VMX_VMCS_GUEST_CR0,
630	VMX_VMCS_GUEST_CR3,
631	VMX_VMCS_GUEST_CR4,
632	VMX_VMCS_GUEST_ES_BASE,
633	VMX_VMCS_GUEST_CS_BASE,
634	VMX_VMCS_GUEST_SS_BASE,
635	VMX_VMCS_GUEST_DS_BASE,
636	VMX_VMCS_GUEST_FS_BASE,
637	VMX_VMCS_GUEST_GS_BASE,
638	VMX_VMCS_GUEST_LDTR_BASE,
639	VMX_VMCS_GUEST_TR_BASE,
640	VMX_VMCS_GUEST_GDTR_BASE,
641	VMX_VMCS_GUEST_IDTR_BASE,
642	VMX_VMCS_GUEST_DR7,
643	VMX_VMCS_GUEST_RSP,
644	VMX_VMCS_GUEST_RIP,
645	VMX_VMCS_GUEST_RFLAGS,
646	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
647	VMX_VMCS_GUEST_SYSENTER_ESP,
648	VMX_VMCS_GUEST_SYSENTER_EIP,
649
650	/* Natural-width host-state fields */
651	VMX_VMCS_HOST_CR0,
652	VMX_VMCS_HOST_CR3,
653	VMX_VMCS_HOST_CR4,
654	VMX_VMCS_HOST_FS_BASE,
655	VMX_VMCS_HOST_GS_BASE,
656	VMX_VMCS_HOST_TR_BASE,
657	VMX_VMCS_HOST_GDTR_BASE,
658	VMX_VMCS_HOST_IDTR_BASE,
659	VMX_VMCS_HOST_SYSENTER_ESP,
660	VMX_VMCS_HOST_SYSENTER_EIP,
661	VMX_VMCS_HOST_RSP,
662	VMX_VMCS_HOST_RIP
663	};
664	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
665
666	static const uint32_t g_aVmcsSegBase[] =
667	{
668	VMX_VMCS_GUEST_ES_BASE,
669	VMX_VMCS_GUEST_CS_BASE,
670	VMX_VMCS_GUEST_SS_BASE,
671	VMX_VMCS_GUEST_DS_BASE,
672	VMX_VMCS_GUEST_FS_BASE,
673	VMX_VMCS_GUEST_GS_BASE
674	};
675	static const uint32_t g_aVmcsSegSel[] =
676	{
677	VMX_VMCS16_GUEST_ES_SEL,
678	VMX_VMCS16_GUEST_CS_SEL,
679	VMX_VMCS16_GUEST_SS_SEL,
680	VMX_VMCS16_GUEST_DS_SEL,
681	VMX_VMCS16_GUEST_FS_SEL,
682	VMX_VMCS16_GUEST_GS_SEL
683	};
684	static const uint32_t g_aVmcsSegLimit[] =
685	{
686	VMX_VMCS32_GUEST_ES_LIMIT,
687	VMX_VMCS32_GUEST_CS_LIMIT,
688	VMX_VMCS32_GUEST_SS_LIMIT,
689	VMX_VMCS32_GUEST_DS_LIMIT,
690	VMX_VMCS32_GUEST_FS_LIMIT,
691	VMX_VMCS32_GUEST_GS_LIMIT
692	};
693	static const uint32_t g_aVmcsSegAttr[] =
694	{
695	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
696	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
697	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
698	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
699	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
700	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
701	};
702	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
703	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
704	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
705	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
706
707	#ifdef HMVMX_USE_FUNCTION_TABLE
708	/**
709	* VMX_EXIT dispatch table.
710	*/
711	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
712	{
713	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
714	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
715	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
716	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
717	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
718	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
719	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
720	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
721	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
722	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
723	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
724	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
725	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
726	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
727	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
728	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
729	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
730	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
731	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
732	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
733	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
734	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
735	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
736	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
737	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
738	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
739	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
740	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
741	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
742	#else
743	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
744	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
745	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
746	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
747	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
748	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
749	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
750	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
751	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
752	#endif
753	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
754	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
755	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
756	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
757	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
758	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
759	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
760	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
761	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
762	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
763	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
764	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
765	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
766	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
767	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
768	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
769	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
770	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
771	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
772	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
773	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
774	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
775	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
776	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
777	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
778	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
779	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
780	#else
781	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
782	#endif
783	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
784	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
785	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
786	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
787	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
788	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
789	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
790	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
791	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
792	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
793	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
794	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
795	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
796	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
797	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
798	};
799	#endif /* HMVMX_USE_FUNCTION_TABLE */
800
801	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
802	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
803	{
804	/* 0 */ "(Not Used)",
805	/* 1 */ "VMCALL executed in VMX root operation.",
806	/* 2 */ "VMCLEAR with invalid physical address.",
807	/* 3 */ "VMCLEAR with VMXON pointer.",
808	/* 4 */ "VMLAUNCH with non-clear VMCS.",
809	/* 5 */ "VMRESUME with non-launched VMCS.",
810	/* 6 */ "VMRESUME after VMXOFF",
811	/* 7 */ "VM-entry with invalid control fields.",
812	/* 8 */ "VM-entry with invalid host state fields.",
813	/* 9 */ "VMPTRLD with invalid physical address.",
814	/* 10 */ "VMPTRLD with VMXON pointer.",
815	/* 11 */ "VMPTRLD with incorrect revision identifier.",
816	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
817	/* 13 */ "VMWRITE to read-only VMCS component.",
818	/* 14 */ "(Not Used)",
819	/* 15 */ "VMXON executed in VMX root operation.",
820	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
821	/* 17 */ "VM-entry with non-launched executing VMCS.",
822	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
823	/* 19 */ "VMCALL with non-clear VMCS.",
824	/* 20 */ "VMCALL with invalid VM-exit control fields.",
825	/* 21 */ "(Not Used)",
826	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
827	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
828	/* 24 */ "VMCALL with invalid SMM-monitor features.",
829	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
830	/* 26 */ "VM-entry with events blocked by MOV SS.",
831	/* 27 */ "(Not Used)",
832	/* 28 */ "Invalid operand to INVEPT/INVVPID."
833	};
834	#endif /* VBOX_STRICT && LOG_ENABLED */
835
836
837	/**
838	* Gets the CR0 guest/host mask.
839	*
840	* These bits typically does not change through the lifetime of a VM. Any bit set in
841	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
842	* by the guest.
843	*
844	* @returns The CR0 guest/host mask.
845	* @param pVCpu The cross context virtual CPU structure.
846	*/
847	static uint64_t hmR0VmxGetFixedCr0Mask(PCVMCPUCC pVCpu)
848	{
849	/*
850	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
851	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
852	*
853	* Furthermore, modifications to any bits that are reserved/unspecified currently
854	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
855	* when future CPUs specify and use currently reserved/unspecified bits.
856	*/
857	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
858	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
859	* and @bugref{6944}. */
860	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
861	return ( X86_CR0_PE
862	\| X86_CR0_NE
863	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
864	\| X86_CR0_PG
865	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
866	}
867
868
869	/**
870	* Gets the CR4 guest/host mask.
871	*
872	* These bits typically does not change through the lifetime of a VM. Any bit set in
873	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
874	* by the guest.
875	*
876	* @returns The CR4 guest/host mask.
877	* @param pVCpu The cross context virtual CPU structure.
878	*/
879	static uint64_t hmR0VmxGetFixedCr4Mask(PCVMCPUCC pVCpu)
880	{
881	/*
882	* We construct a mask of all CR4 bits that the guest can modify without causing
883	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
884	* a VM-exit when the guest attempts to modify them when executing using
885	* hardware-assisted VMX.
886	*
887	* When a feature is not exposed to the guest (and may be present on the host),
888	* we want to intercept guest modifications to the bit so we can emulate proper
889	* behavior (e.g., #GP).
890	*
891	* Furthermore, only modifications to those bits that don't require immediate
892	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
893	* depends on CR3 which might not always be the guest value while executing
894	* using hardware-assisted VMX.
895	*/
896	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
897	bool const fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
898	bool const fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
899	bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
900
901	/*
902	* Paranoia.
903	* Ensure features exposed to the guest are present on the host.
904	*/
905	Assert(!fFsGsBase \|\| pVM->cpum.ro.HostFeatures.fFsGsBase);
906	Assert(!fXSaveRstor \|\| pVM->cpum.ro.HostFeatures.fXSaveRstor);
907	Assert(!fFxSaveRstor \|\| pVM->cpum.ro.HostFeatures.fFxSaveRstor);
908
909	uint64_t const fGstMask = ( X86_CR4_PVI
910	\| X86_CR4_TSD
911	\| X86_CR4_DE
912	\| X86_CR4_MCE
913	\| X86_CR4_PCE
914	\| X86_CR4_OSXMMEEXCPT
915	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
916	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
917	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0));
918	return ~fGstMask;
919	}
920
921
922	/**
923	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
924	* area.
925	*
926	* @returns @c true if it's different, @c false otherwise.
927	* @param pVmcsInfo The VMCS info. object.
928	*/
929	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
930	{
931	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
932	&& pVmcsInfo->pvGuestMsrStore);
933	}
934
935
936	/**
937	* Sets the given Processor-based VM-execution controls.
938	*
939	* @param pVmxTransient The VMX-transient structure.
940	* @param uProcCtls The Processor-based VM-execution controls to set.
941	*/
942	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
943	{
944	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
945	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
946	{
947	pVmcsInfo->u32ProcCtls \|= uProcCtls;
948	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
949	AssertRC(rc);
950	}
951	}
952
953
954	/**
955	* Removes the given Processor-based VM-execution controls.
956	*
957	* @param pVCpu The cross context virtual CPU structure.
958	* @param pVmxTransient The VMX-transient structure.
959	* @param uProcCtls The Processor-based VM-execution controls to remove.
960	*
961	* @remarks When executing a nested-guest, this will not remove any of the specified
962	* controls if the nested hypervisor has set any one of them.
963	*/
964	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
965	{
966	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
967	if (pVmcsInfo->u32ProcCtls & uProcCtls)
968	{
969	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
970	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
971	? true
972	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
973	#else
974	NOREF(pVCpu);
975	bool const fRemoveCtls = true;
976	#endif
977	if (fRemoveCtls)
978	{
979	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
980	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
981	AssertRC(rc);
982	}
983	}
984	}
985
986
987	/**
988	* Sets the TSC offset for the current VMCS.
989	*
990	* @param uTscOffset The TSC offset to set.
991	* @param pVmcsInfo The VMCS info. object.
992	*/
993	static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
994	{
995	if (pVmcsInfo->u64TscOffset != uTscOffset)
996	{
997	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
998	AssertRC(rc);
999	pVmcsInfo->u64TscOffset = uTscOffset;
1000	}
1001	}
1002
1003
1004	/**
1005	* Adds one or more exceptions to the exception bitmap and commits it to the current
1006	* VMCS.
1007	*
1008	* @param pVmxTransient The VMX-transient structure.
1009	* @param uXcptMask The exception(s) to add.
1010	*/
1011	static void hmR0VmxAddXcptInterceptMask(PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1012	{
1013	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1014	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1015	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1016	{
1017	uXcptBitmap \|= uXcptMask;
1018	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1019	AssertRC(rc);
1020	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1021	}
1022	}
1023
1024
1025	/**
1026	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1027	*
1028	* @param pVmxTransient The VMX-transient structure.
1029	* @param uXcpt The exception to add.
1030	*/
1031	static void hmR0VmxAddXcptIntercept(PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1032	{
1033	Assert(uXcpt <= X86_XCPT_LAST);
1034	hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1035	}
1036
1037
1038	/**
1039	* Remove one or more exceptions from the exception bitmap and commits it to the
1040	* current VMCS.
1041	*
1042	* This takes care of not removing the exception intercept if a nested-guest
1043	* requires the exception to be intercepted.
1044	*
1045	* @returns VBox status code.
1046	* @param pVCpu The cross context virtual CPU structure.
1047	* @param pVmxTransient The VMX-transient structure.
1048	* @param uXcptMask The exception(s) to remove.
1049	*/
1050	static int hmR0VmxRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1051	{
1052	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1053	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1054	if (u32XcptBitmap & uXcptMask)
1055	{
1056	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1057	if (!pVmxTransient->fIsNestedGuest)
1058	{ /* likely */ }
1059	else
1060	{
1061	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1062	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1063	}
1064	#endif
1065	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1066	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1067	\| RT_BIT(X86_XCPT_DE)
1068	\| RT_BIT(X86_XCPT_NM)
1069	\| RT_BIT(X86_XCPT_TS)
1070	\| RT_BIT(X86_XCPT_UD)
1071	\| RT_BIT(X86_XCPT_NP)
1072	\| RT_BIT(X86_XCPT_SS)
1073	\| RT_BIT(X86_XCPT_GP)
1074	\| RT_BIT(X86_XCPT_PF)
1075	\| RT_BIT(X86_XCPT_MF));
1076	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1077	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1078	#endif
1079	if (uXcptMask)
1080	{
1081	/* Validate we are not removing any essential exception intercepts. */
1082	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1083	NOREF(pVCpu);
1084	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1085	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1086
1087	/* Remove it from the exception bitmap. */
1088	u32XcptBitmap &= ~uXcptMask;
1089
1090	/* Commit and update the cache if necessary. */
1091	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1092	{
1093	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1094	AssertRC(rc);
1095	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1096	}
1097	}
1098	}
1099	return VINF_SUCCESS;
1100	}
1101
1102
1103	/**
1104	* Remove an exceptions from the exception bitmap and commits it to the current
1105	* VMCS.
1106	*
1107	* @returns VBox status code.
1108	* @param pVCpu The cross context virtual CPU structure.
1109	* @param pVmxTransient The VMX-transient structure.
1110	* @param uXcpt The exception to remove.
1111	*/
1112	static int hmR0VmxRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1113	{
1114	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1115	}
1116
1117
1118	/**
1119	* Loads the VMCS specified by the VMCS info. object.
1120	*
1121	* @returns VBox status code.
1122	* @param pVmcsInfo The VMCS info. object.
1123	*
1124	* @remarks Can be called with interrupts disabled.
1125	*/
1126	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1127	{
1128	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1129	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1130
1131	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1132	if (RT_SUCCESS(rc))
1133	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1134	return rc;
1135	}
1136
1137
1138	/**
1139	* Clears the VMCS specified by the VMCS info. object.
1140	*
1141	* @returns VBox status code.
1142	* @param pVmcsInfo The VMCS info. object.
1143	*
1144	* @remarks Can be called with interrupts disabled.
1145	*/
1146	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1147	{
1148	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1150
1151	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1152	if (RT_SUCCESS(rc))
1153	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1154	return rc;
1155	}
1156
1157
1158	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1159	/**
1160	* Loads the shadow VMCS specified by the VMCS info. object.
1161	*
1162	* @returns VBox status code.
1163	* @param pVmcsInfo The VMCS info. object.
1164	*
1165	* @remarks Can be called with interrupts disabled.
1166	*/
1167	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1168	{
1169	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1170	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1171
1172	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1173	if (RT_SUCCESS(rc))
1174	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1175	return rc;
1176	}
1177
1178
1179	/**
1180	* Clears the shadow VMCS specified by the VMCS info. object.
1181	*
1182	* @returns VBox status code.
1183	* @param pVmcsInfo The VMCS info. object.
1184	*
1185	* @remarks Can be called with interrupts disabled.
1186	*/
1187	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1188	{
1189	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1190	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1191
1192	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1193	if (RT_SUCCESS(rc))
1194	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1195	return rc;
1196	}
1197
1198
1199	/**
1200	* Switches from and to the specified VMCSes.
1201	*
1202	* @returns VBox status code.
1203	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1204	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1205	*
1206	* @remarks Called with interrupts disabled.
1207	*/
1208	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1209	{
1210	/*
1211	* Clear the VMCS we are switching out if it has not already been cleared.
1212	* This will sync any CPU internal data back to the VMCS.
1213	*/
1214	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1215	{
1216	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1217	if (RT_SUCCESS(rc))
1218	{
1219	/*
1220	* The shadow VMCS, if any, would not be active at this point since we
1221	* would have cleared it while importing the virtual hardware-virtualization
1222	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1223	* clear the shadow VMCS here, just assert for safety.
1224	*/
1225	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1226	}
1227	else
1228	return rc;
1229	}
1230
1231	/*
1232	* Clear the VMCS we are switching to if it has not already been cleared.
1233	* This will initialize the VMCS launch state to "clear" required for loading it.
1234	*
1235	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1236	*/
1237	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1238	{
1239	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1240	if (RT_SUCCESS(rc))
1241	{ /* likely */ }
1242	else
1243	return rc;
1244	}
1245
1246	/*
1247	* Finally, load the VMCS we are switching to.
1248	*/
1249	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1250	}
1251
1252
1253	/**
1254	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1255	* caller.
1256	*
1257	* @returns VBox status code.
1258	* @param pVCpu The cross context virtual CPU structure.
1259	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1260	* true) or guest VMCS (pass false).
1261	*/
1262	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1263	{
1264	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1265	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1266
1267	PVMXVMCSINFO pVmcsInfoFrom;
1268	PVMXVMCSINFO pVmcsInfoTo;
1269	if (fSwitchToNstGstVmcs)
1270	{
1271	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1272	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1273	}
1274	else
1275	{
1276	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1277	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1278	}
1279
1280	/*
1281	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1282	* preemption hook code path acquires the current VMCS.
1283	*/
1284	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1285
1286	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1287	if (RT_SUCCESS(rc))
1288	{
1289	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1290
1291	/*
1292	* If we are switching to a VMCS that was executed on a different host CPU or was
1293	* never executed before, flag that we need to export the host state before executing
1294	* guest/nested-guest code using hardware-assisted VMX.
1295	*
1296	* This could probably be done in a preemptible context since the preemption hook
1297	* will flag the necessary change in host context. However, since preemption is
1298	* already disabled and to avoid making assumptions about host specific code in
1299	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1300	* disabled.
1301	*/
1302	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1303	{ /* likely */ }
1304	else
1305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1306
1307	ASMSetFlags(fEFlags);
1308
1309	/*
1310	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1311	* flag that we need to update the host MSR values there. Even if we decide in the
1312	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1313	* if its content differs, we would have to update the host MSRs anyway.
1314	*/
1315	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1316	}
1317	else
1318	ASMSetFlags(fEFlags);
1319	return rc;
1320	}
1321	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1322
1323
1324	/**
1325	* Updates the VM's last error record.
1326	*
1327	* If there was a VMX instruction error, reads the error data from the VMCS and
1328	* updates VCPU's last error record as well.
1329	*
1330	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1331	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1332	* VERR_VMX_INVALID_VMCS_FIELD.
1333	* @param rc The error code.
1334	*/
1335	static void hmR0VmxUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
1336	{
1337	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1338	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1339	{
1340	AssertPtrReturnVoid(pVCpu);
1341	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1342	}
1343	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1344	}
1345
1346
1347	#ifdef VBOX_STRICT
1348	/**
1349	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1350	* transient structure.
1351	*
1352	* @param pVmxTransient The VMX-transient structure.
1353	*/
1354	DECLINLINE(void) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1355	{
1356	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1357	AssertRC(rc);
1358	}
1359
1360
1361	/**
1362	* Reads the VM-entry exception error code field from the VMCS into
1363	* the VMX transient structure.
1364	*
1365	* @param pVmxTransient The VMX-transient structure.
1366	*/
1367	DECLINLINE(void) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1368	{
1369	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1370	AssertRC(rc);
1371	}
1372
1373
1374	/**
1375	* Reads the VM-entry exception error code field from the VMCS into
1376	* the VMX transient structure.
1377	*
1378	* @param pVmxTransient The VMX-transient structure.
1379	*/
1380	DECLINLINE(void) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1381	{
1382	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1383	AssertRC(rc);
1384	}
1385	#endif /* VBOX_STRICT */
1386
1387
1388	/**
1389	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1390	* transient structure.
1391	*
1392	* @param pVmxTransient The VMX-transient structure.
1393	*/
1394	DECLINLINE(void) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1395	{
1396	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1397	{
1398	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1399	AssertRC(rc);
1400	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1401	}
1402	}
1403
1404
1405	/**
1406	* Reads the VM-exit interruption error code from the VMCS into the VMX
1407	* transient structure.
1408	*
1409	* @param pVmxTransient The VMX-transient structure.
1410	*/
1411	DECLINLINE(void) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1412	{
1413	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1414	{
1415	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1416	AssertRC(rc);
1417	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1418	}
1419	}
1420
1421
1422	/**
1423	* Reads the VM-exit instruction length field from the VMCS into the VMX
1424	* transient structure.
1425	*
1426	* @param pVmxTransient The VMX-transient structure.
1427	*/
1428	DECLINLINE(void) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1429	{
1430	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1431	{
1432	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1433	AssertRC(rc);
1434	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1435	}
1436	}
1437
1438
1439	/**
1440	* Reads the VM-exit instruction-information field from the VMCS into
1441	* the VMX transient structure.
1442	*
1443	* @param pVmxTransient The VMX-transient structure.
1444	*/
1445	DECLINLINE(void) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1446	{
1447	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1448	{
1449	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1450	AssertRC(rc);
1451	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1452	}
1453	}
1454
1455
1456	/**
1457	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1458	*
1459	* @param pVmxTransient The VMX-transient structure.
1460	*/
1461	DECLINLINE(void) hmR0VmxReadExitQualVmcs(PVMXTRANSIENT pVmxTransient)
1462	{
1463	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1464	{
1465	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1466	AssertRC(rc);
1467	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1468	}
1469	}
1470
1471
1472	/**
1473	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1474	*
1475	* @param pVmxTransient The VMX-transient structure.
1476	*/
1477	DECLINLINE(void) hmR0VmxReadGuestLinearAddrVmcs(PVMXTRANSIENT pVmxTransient)
1478	{
1479	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1480	{
1481	int rc = VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1482	AssertRC(rc);
1483	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1484	}
1485	}
1486
1487
1488	/**
1489	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1490	*
1491	* @param pVmxTransient The VMX-transient structure.
1492	*/
1493	DECLINLINE(void) hmR0VmxReadGuestPhysicalAddrVmcs(PVMXTRANSIENT pVmxTransient)
1494	{
1495	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1496	{
1497	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1498	AssertRC(rc);
1499	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1500	}
1501	}
1502
1503	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1504	/**
1505	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1506	* structure.
1507	*
1508	* @param pVmxTransient The VMX-transient structure.
1509	*/
1510	DECLINLINE(void) hmR0VmxReadGuestPendingDbgXctps(PVMXTRANSIENT pVmxTransient)
1511	{
1512	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1513	{
1514	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1515	AssertRC(rc);
1516	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1517	}
1518	}
1519	#endif
1520
1521	/**
1522	* Reads the IDT-vectoring information field from the VMCS into the VMX
1523	* transient structure.
1524	*
1525	* @param pVmxTransient The VMX-transient structure.
1526	*
1527	* @remarks No-long-jump zone!!!
1528	*/
1529	DECLINLINE(void) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1530	{
1531	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1532	{
1533	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1534	AssertRC(rc);
1535	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1536	}
1537	}
1538
1539
1540	/**
1541	* Reads the IDT-vectoring error code from the VMCS into the VMX
1542	* transient structure.
1543	*
1544	* @param pVmxTransient The VMX-transient structure.
1545	*/
1546	DECLINLINE(void) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1547	{
1548	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1549	{
1550	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1551	AssertRC(rc);
1552	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1553	}
1554	}
1555
1556	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1557	/**
1558	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1559	*
1560	* @param pVmxTransient The VMX-transient structure.
1561	*/
1562	static void hmR0VmxReadAllRoFieldsVmcs(PVMXTRANSIENT pVmxTransient)
1563	{
1564	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1565	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1566	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1567	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1568	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1569	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1570	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1571	rc \|= VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1572	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1573	AssertRC(rc);
1574	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1575	\| HMVMX_READ_EXIT_INSTR_LEN
1576	\| HMVMX_READ_EXIT_INSTR_INFO
1577	\| HMVMX_READ_IDT_VECTORING_INFO
1578	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1579	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1580	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1581	\| HMVMX_READ_GUEST_LINEAR_ADDR
1582	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1583	}
1584	#endif
1585
1586	/**
1587	* Enters VMX root mode operation on the current CPU.
1588	*
1589	* @returns VBox status code.
1590	* @param pHostCpu The HM physical-CPU structure.
1591	* @param pVM The cross context VM structure. Can be
1592	* NULL, after a resume.
1593	* @param HCPhysCpuPage Physical address of the VMXON region.
1594	* @param pvCpuPage Pointer to the VMXON region.
1595	*/
1596	static int hmR0VmxEnterRootMode(PHMPHYSCPU pHostCpu, PVMCC pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1597	{
1598	Assert(pHostCpu);
1599	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1600	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1601	Assert(pvCpuPage);
1602	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1603
1604	if (pVM)
1605	{
1606	/* Write the VMCS revision identifier to the VMXON region. */
1607	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1608	}
1609
1610	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1611	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1612
1613	/* Enable the VMX bit in CR4 if necessary. */
1614	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1615
1616	/* Record whether VMXE was already prior to us enabling it above. */
1617	pHostCpu->fVmxeAlreadyEnabled = RT_BOOL(uOldCr4 & X86_CR4_VMXE);
1618
1619	/* Enter VMX root mode. */
1620	int rc = VMXEnable(HCPhysCpuPage);
1621	if (RT_FAILURE(rc))
1622	{
1623	/* Restore CR4.VMXE if it was not set prior to our attempt to set it above. */
1624	if (!pHostCpu->fVmxeAlreadyEnabled)
1625	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1626
1627	if (pVM)
1628	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1629	}
1630
1631	/* Restore interrupts. */
1632	ASMSetFlags(fEFlags);
1633	return rc;
1634	}
1635
1636
1637	/**
1638	* Exits VMX root mode operation on the current CPU.
1639	*
1640	* @returns VBox status code.
1641	* @param pHostCpu The HM physical-CPU structure.
1642	*/
1643	static int hmR0VmxLeaveRootMode(PHMPHYSCPU pHostCpu)
1644	{
1645	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1646
1647	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1648	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1649
1650	/* If we're for some reason not in VMX root mode, then don't leave it. */
1651	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1652
1653	int rc;
1654	if (uHostCr4 & X86_CR4_VMXE)
1655	{
1656	/* Exit VMX root mode and clear the VMX bit in CR4. */
1657	VMXDisable();
1658
1659	/* Clear CR4.VMXE only if it was clear prior to use setting it. */
1660	if (!pHostCpu->fVmxeAlreadyEnabled)
1661	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1662
1663	rc = VINF_SUCCESS;
1664	}
1665	else
1666	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1667
1668	/* Restore interrupts. */
1669	ASMSetFlags(fEFlags);
1670	return rc;
1671	}
1672
1673
1674	/**
1675	* Allocates and maps a physically contiguous page. The allocated page is
1676	* zero'd out (used by various VT-x structures).
1677	*
1678	* @returns IPRT status code.
1679	* @param pMemObj Pointer to the ring-0 memory object.
1680	* @param ppVirt Where to store the virtual address of the allocation.
1681	* @param pHCPhys Where to store the physical address of the allocation.
1682	*/
1683	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1684	{
1685	AssertPtr(pMemObj);
1686	AssertPtr(ppVirt);
1687	AssertPtr(pHCPhys);
1688	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1689	if (RT_FAILURE(rc))
1690	return rc;
1691	ppVirt = RTR0MemObjAddress(pMemObj);
1692	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1693	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1694	return VINF_SUCCESS;
1695	}
1696
1697
1698	/**
1699	* Frees and unmaps an allocated, physical page.
1700	*
1701	* @param pMemObj Pointer to the ring-0 memory object.
1702	* @param ppVirt Where to re-initialize the virtual address of allocation as
1703	* 0.
1704	* @param pHCPhys Where to re-initialize the physical address of the
1705	* allocation as 0.
1706	*/
1707	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1708	{
1709	AssertPtr(pMemObj);
1710	AssertPtr(ppVirt);
1711	AssertPtr(pHCPhys);
1712	/* NULL is valid, accepted and ignored by the free function below. */
1713	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1714	*pMemObj = NIL_RTR0MEMOBJ;
1715	*ppVirt = NULL;
1716	*pHCPhys = NIL_RTHCPHYS;
1717	}
1718
1719
1720	/**
1721	* Initializes a VMCS info. object.
1722	*
1723	* @param pVmcsInfo The VMCS info. object.
1724	*/
1725	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1726	{
1727	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1728
1729	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1730	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1731	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1732	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1733	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1734	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1735	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1736	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1737	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1738	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1739	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1740	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1741	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1742	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1743	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1744	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
1745	pVmcsInfo->idHostCpuExec = NIL_RTCPUID;
1746	}
1747
1748
1749	/**
1750	* Frees the VT-x structures for a VMCS info. object.
1751	*
1752	* @param pVM The cross context VM structure.
1753	* @param pVmcsInfo The VMCS info. object.
1754	*/
1755	static void hmR0VmxFreeVmcsInfo(PVMCC pVM, PVMXVMCSINFO pVmcsInfo)
1756	{
1757	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1758
1759	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1760	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1761	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1762	#endif
1763
1764	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1765	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1766
1767	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1768	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1769	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1770
1771	hmR0VmxInitVmcsInfo(pVmcsInfo);
1772	}
1773
1774
1775	/**
1776	* Allocates the VT-x structures for a VMCS info. object.
1777	*
1778	* @returns VBox status code.
1779	* @param pVCpu The cross context virtual CPU structure.
1780	* @param pVmcsInfo The VMCS info. object.
1781	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1782	*/
1783	static int hmR0VmxAllocVmcsInfo(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1784	{
1785	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1786
1787	/* Allocate the guest VM control structure (VMCS). */
1788	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1789	if (RT_SUCCESS(rc))
1790	{
1791	if (!fIsNstGstVmcs)
1792	{
1793	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1794	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1795	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1796	#endif
1797	if (RT_SUCCESS(rc))
1798	{
1799	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1800	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1801	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1802	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1803	}
1804	}
1805	else
1806	{
1807	/* We don't yet support exposing VMCS shadowing to the guest. */
1808	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1809	Assert(!pVmcsInfo->pvShadowVmcs);
1810
1811	/* Get the allocated virtual-APIC page from CPUM. */
1812	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1813	{
1814	/** @todo NSTVMX: Get rid of this. There is no need to allocate a separate HC
1815	* page for this. Use the one provided by the nested-guest directly. */
1816	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1817	&pVmcsInfo->HCPhysVirtApic);
1818	Assert(pVmcsInfo->pbVirtApic);
1819	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1820	}
1821	}
1822
1823	if (RT_SUCCESS(rc))
1824	{
1825	/*
1826	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1827	* transparent accesses of specific MSRs.
1828	*
1829	* If the condition for enabling MSR bitmaps changes here, don't forget to
1830	* update HMIsMsrBitmapActive().
1831	*
1832	* We don't share MSR bitmaps between the guest and nested-guest as we then
1833	* don't need to care about carefully restoring the guest MSR bitmap.
1834	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1835	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1836	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1837	* we do that later while merging VMCS.
1838	*/
1839	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1840	{
1841	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1842	if ( RT_SUCCESS(rc)
1843	&& !fIsNstGstVmcs)
1844	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1845	}
1846
1847	if (RT_SUCCESS(rc))
1848	{
1849	/*
1850	* Allocate the VM-entry MSR-load area for the guest MSRs.
1851	*
1852	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1853	* the guest and nested-guest.
1854	*/
1855	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1856	&pVmcsInfo->HCPhysGuestMsrLoad);
1857	if (RT_SUCCESS(rc))
1858	{
1859	/*
1860	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1861	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1862	*/
1863	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1864	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1865	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1866
1867	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1868	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1869	&pVmcsInfo->HCPhysHostMsrLoad);
1870	}
1871	}
1872	}
1873	}
1874
1875	return rc;
1876	}
1877
1878
1879	/**
1880	* Free all VT-x structures for the VM.
1881	*
1882	* @returns IPRT status code.
1883	* @param pVM The cross context VM structure.
1884	*/
1885	static void hmR0VmxStructsFree(PVMCC pVM)
1886	{
1887	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1888	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1889	#endif
1890	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1891
1892	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1893	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1894	{
1895	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1896	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1897	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1898	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1899	}
1900	#endif
1901
1902	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1903	{
1904	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1905	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1906	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1907	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1908	if (pVM->cpum.ro.GuestFeatures.fVmx)
1909	{
1910	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1911	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1912	}
1913	#endif
1914	}
1915	}
1916
1917
1918	/**
1919	* Allocate all VT-x structures for the VM.
1920	*
1921	* @returns IPRT status code.
1922	* @param pVM The cross context VM structure.
1923	*/
1924	static int hmR0VmxStructsAlloc(PVMCC pVM)
1925	{
1926	/*
1927	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1928	* The VMCS size cannot be more than 4096 bytes.
1929	*
1930	* See Intel spec. Appendix A.1 "Basic VMX Information".
1931	*/
1932	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1933	if (cbVmcs <= X86_PAGE_4K_SIZE)
1934	{ /* likely */ }
1935	else
1936	{
1937	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1938	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1939	}
1940
1941	/*
1942	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1943	*/
1944	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1945	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1946	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1947	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1948	#endif
1949
1950	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1951	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1952	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1953
1954	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1955	{
1956	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1957	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1958	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1959	}
1960
1961	/*
1962	* Allocate per-VM VT-x structures.
1963	*/
1964	int rc = VINF_SUCCESS;
1965	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1966	/* Allocate crash-dump magic scratch page. */
1967	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1968	if (RT_FAILURE(rc))
1969	{
1970	hmR0VmxStructsFree(pVM);
1971	return rc;
1972	}
1973	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1974	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1975	#endif
1976
1977	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1978	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1979	{
1980	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1981	&pVM->hm.s.vmx.HCPhysApicAccess);
1982	if (RT_FAILURE(rc))
1983	{
1984	hmR0VmxStructsFree(pVM);
1985	return rc;
1986	}
1987	}
1988
1989	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1990	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1991	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1992	{
1993	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1994	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1995	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1996	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1997	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1998	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1999	{
2000	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
2001	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
2002	if (RT_SUCCESS(rc))
2003	{
2004	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
2005	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
2006	}
2007	}
2008	else
2009	rc = VERR_NO_MEMORY;
2010
2011	if (RT_FAILURE(rc))
2012	{
2013	hmR0VmxStructsFree(pVM);
2014	return rc;
2015	}
2016	}
2017	#endif
2018
2019	/*
2020	* Initialize per-VCPU VT-x structures.
2021	*/
2022	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2023	{
2024	/* Allocate the guest VMCS structures. */
2025	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
2026	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2027	if (RT_SUCCESS(rc))
2028	{
2029	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2030	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2031	if (pVM->cpum.ro.GuestFeatures.fVmx)
2032	{
2033	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2034	if (RT_SUCCESS(rc))
2035	{ /* likely */ }
2036	else
2037	break;
2038	}
2039	#endif
2040	}
2041	else
2042	break;
2043	}
2044
2045	if (RT_FAILURE(rc))
2046	{
2047	hmR0VmxStructsFree(pVM);
2048	return rc;
2049	}
2050
2051	return VINF_SUCCESS;
2052	}
2053
2054	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2055	/**
2056	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2057	*
2058	* @returns @c true if the MSR is intercepted, @c false otherwise.
2059	* @param pvMsrBitmap The MSR bitmap.
2060	* @param offMsr The MSR byte offset.
2061	* @param iBit The bit offset from the byte offset.
2062	*/
2063	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2064	{
2065	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2066	Assert(pbMsrBitmap);
2067	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2068	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2069	}
2070	#endif
2071
2072	/**
2073	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2074	*
2075	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2076	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2077	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2078	* the read/write access of this MSR.
2079	*
2080	* @param pVCpu The cross context virtual CPU structure.
2081	* @param pVmcsInfo The VMCS info. object.
2082	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2083	* @param idMsr The MSR value.
2084	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2085	* include both a read -and- a write permission!
2086	*
2087	* @sa CPUMGetVmxMsrPermission.
2088	* @remarks Can be called with interrupts disabled.
2089	*/
2090	static void hmR0VmxSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2091	{
2092	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2093	Assert(pbMsrBitmap);
2094	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2095
2096	/*
2097	* MSR-bitmap Layout:
2098	* Byte index MSR range Interpreted as
2099	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2100	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2101	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2102	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2103	*
2104	* A bit corresponding to an MSR within the above range causes a VM-exit
2105	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2106	* the MSR range, it always cause a VM-exit.
2107	*
2108	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2109	*/
2110	uint16_t const offBitmapRead = 0;
2111	uint16_t const offBitmapWrite = 0x800;
2112	uint16_t offMsr;
2113	int32_t iBit;
2114	if (idMsr <= UINT32_C(0x00001fff))
2115	{
2116	offMsr = 0;
2117	iBit = idMsr;
2118	}
2119	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2120	{
2121	offMsr = 0x400;
2122	iBit = idMsr - UINT32_C(0xc0000000);
2123	}
2124	else
2125	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2126
2127	/*
2128	* Set the MSR read permission.
2129	*/
2130	uint16_t const offMsrRead = offBitmapRead + offMsr;
2131	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2132	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2133	{
2134	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2135	bool const fClear = !fIsNstGstVmcs ? true
2136	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2137	#else
2138	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2139	bool const fClear = true;
2140	#endif
2141	if (fClear)
2142	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2143	}
2144	else
2145	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2146
2147	/*
2148	* Set the MSR write permission.
2149	*/
2150	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2151	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2152	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2153	{
2154	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2155	bool const fClear = !fIsNstGstVmcs ? true
2156	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2157	#else
2158	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2159	bool const fClear = true;
2160	#endif
2161	if (fClear)
2162	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2163	}
2164	else
2165	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2166	}
2167
2168
2169	/**
2170	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2171	* area.
2172	*
2173	* @returns VBox status code.
2174	* @param pVCpu The cross context virtual CPU structure.
2175	* @param pVmcsInfo The VMCS info. object.
2176	* @param cMsrs The number of MSRs.
2177	*/
2178	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2179	{
2180	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2181	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2182	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2183	{
2184	/* Commit the MSR counts to the VMCS and update the cache. */
2185	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2186	{
2187	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2188	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
2189	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2190	pVmcsInfo->cEntryMsrLoad = cMsrs;
2191	pVmcsInfo->cExitMsrStore = cMsrs;
2192	pVmcsInfo->cExitMsrLoad = cMsrs;
2193	}
2194	return VINF_SUCCESS;
2195	}
2196
2197	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2198	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2199	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2200	}
2201
2202
2203	/**
2204	* Adds a new (or updates the value of an existing) guest/host MSR
2205	* pair to be swapped during the world-switch as part of the
2206	* auto-load/store MSR area in the VMCS.
2207	*
2208	* @returns VBox status code.
2209	* @param pVCpu The cross context virtual CPU structure.
2210	* @param pVmxTransient The VMX-transient structure.
2211	* @param idMsr The MSR.
2212	* @param uGuestMsrValue Value of the guest MSR.
2213	* @param fSetReadWrite Whether to set the guest read/write access of this
2214	* MSR (thus not causing a VM-exit).
2215	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2216	* necessary.
2217	*/
2218	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2219	bool fSetReadWrite, bool fUpdateHostMsr)
2220	{
2221	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2222	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2223	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2224	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2225	uint32_t i;
2226
2227	/* Paranoia. */
2228	Assert(pGuestMsrLoad);
2229
2230	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2231
2232	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2233	for (i = 0; i < cMsrs; i++)
2234	{
2235	if (pGuestMsrLoad[i].u32Msr == idMsr)
2236	break;
2237	}
2238
2239	bool fAdded = false;
2240	if (i == cMsrs)
2241	{
2242	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2243	++cMsrs;
2244	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2245	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2246
2247	/* Set the guest to read/write this MSR without causing VM-exits. */
2248	if ( fSetReadWrite
2249	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2250	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2251
2252	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2253	fAdded = true;
2254	}
2255
2256	/* Update the MSR value for the newly added or already existing MSR. */
2257	pGuestMsrLoad[i].u32Msr = idMsr;
2258	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2259
2260	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2261	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2262	{
2263	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2264	pGuestMsrStore[i].u32Msr = idMsr;
2265	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2266	}
2267
2268	/* Update the corresponding slot in the host MSR area. */
2269	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2270	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2271	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2272	pHostMsr[i].u32Msr = idMsr;
2273
2274	/*
2275	* Only if the caller requests to update the host MSR value AND we've newly added the
2276	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2277	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2278	*
2279	* We do this for performance reasons since reading MSRs may be quite expensive.
2280	*/
2281	if (fAdded)
2282	{
2283	if (fUpdateHostMsr)
2284	{
2285	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2286	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2287	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2288	}
2289	else
2290	{
2291	/* Someone else can do the work. */
2292	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2293	}
2294	}
2295	return VINF_SUCCESS;
2296	}
2297
2298
2299	/**
2300	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2301	* auto-load/store MSR area in the VMCS.
2302	*
2303	* @returns VBox status code.
2304	* @param pVCpu The cross context virtual CPU structure.
2305	* @param pVmxTransient The VMX-transient structure.
2306	* @param idMsr The MSR.
2307	*/
2308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2309	{
2310	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2311	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2312	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2313	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2314
2315	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2316
2317	for (uint32_t i = 0; i < cMsrs; i++)
2318	{
2319	/* Find the MSR. */
2320	if (pGuestMsrLoad[i].u32Msr == idMsr)
2321	{
2322	/*
2323	* If it's the last MSR, we only need to reduce the MSR count.
2324	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2325	*/
2326	if (i < cMsrs - 1)
2327	{
2328	/* Remove it from the VM-entry MSR-load area. */
2329	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2330	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2331
2332	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2333	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2334	{
2335	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2336	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2337	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2338	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2339	}
2340
2341	/* Remove it from the VM-exit MSR-load area. */
2342	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2343	Assert(pHostMsr[i].u32Msr == idMsr);
2344	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2345	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2346	}
2347
2348	/* Reduce the count to reflect the removed MSR and bail. */
2349	--cMsrs;
2350	break;
2351	}
2352	}
2353
2354	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2355	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2356	{
2357	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2358	AssertRCReturn(rc, rc);
2359
2360	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2361	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2362	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2363
2364	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2365	return VINF_SUCCESS;
2366	}
2367
2368	return VERR_NOT_FOUND;
2369	}
2370
2371
2372	/**
2373	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2374	*
2375	* @returns @c true if found, @c false otherwise.
2376	* @param pVmcsInfo The VMCS info. object.
2377	* @param idMsr The MSR to find.
2378	*/
2379	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2380	{
2381	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2382	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2383	Assert(pMsrs);
2384	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2385	for (uint32_t i = 0; i < cMsrs; i++)
2386	{
2387	if (pMsrs[i].u32Msr == idMsr)
2388	return true;
2389	}
2390	return false;
2391	}
2392
2393
2394	/**
2395	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2396	*
2397	* @param pVCpu The cross context virtual CPU structure.
2398	* @param pVmcsInfo The VMCS info. object.
2399	*
2400	* @remarks No-long-jump zone!!!
2401	*/
2402	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2403	{
2404	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2405
2406	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2407	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2408	Assert(pHostMsrLoad);
2409	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2410	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2411	for (uint32_t i = 0; i < cMsrs; i++)
2412	{
2413	/*
2414	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2415	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2416	*/
2417	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2418	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2419	else
2420	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2421	}
2422	}
2423
2424
2425	/**
2426	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2427	* perform lazy restoration of the host MSRs while leaving VT-x.
2428	*
2429	* @param pVCpu The cross context virtual CPU structure.
2430	*
2431	* @remarks No-long-jump zone!!!
2432	*/
2433	static void hmR0VmxLazySaveHostMsrs(PVMCPUCC pVCpu)
2434	{
2435	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2436
2437	/*
2438	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2439	*/
2440	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2441	{
2442	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2443	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2444	{
2445	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2446	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2447	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2448	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2449	}
2450	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2451	}
2452	}
2453
2454
2455	/**
2456	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2457	* lazily while leaving VT-x.
2458	*
2459	* @returns true if it does, false otherwise.
2460	* @param pVCpu The cross context virtual CPU structure.
2461	* @param idMsr The MSR to check.
2462	*/
2463	static bool hmR0VmxIsLazyGuestMsr(PCVMCPUCC pVCpu, uint32_t idMsr)
2464	{
2465	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2466	{
2467	switch (idMsr)
2468	{
2469	case MSR_K8_LSTAR:
2470	case MSR_K6_STAR:
2471	case MSR_K8_SF_MASK:
2472	case MSR_K8_KERNEL_GS_BASE:
2473	return true;
2474	}
2475	}
2476	return false;
2477	}
2478
2479
2480	/**
2481	* Loads a set of guests MSRs to allow read/passthru to the guest.
2482	*
2483	* The name of this function is slightly confusing. This function does NOT
2484	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2485	* common prefix for functions dealing with "lazy restoration" of the shared
2486	* MSRs.
2487	*
2488	* @param pVCpu The cross context virtual CPU structure.
2489	*
2490	* @remarks No-long-jump zone!!!
2491	*/
2492	static void hmR0VmxLazyLoadGuestMsrs(PVMCPUCC pVCpu)
2493	{
2494	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2495	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2496
2497	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2498	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2499	{
2500	/*
2501	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2502	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2503	* we can skip a few MSR writes.
2504	*
2505	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2506	* guest MSR values in the guest-CPU context might be different to what's currently
2507	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2508	* CPU, see @bugref{8728}.
2509	*/
2510	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2511	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2512	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2513	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2514	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2515	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2516	{
2517	#ifdef VBOX_STRICT
2518	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2519	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2520	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2521	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2522	#endif
2523	}
2524	else
2525	{
2526	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2527	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2528	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2529	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2530	}
2531	}
2532	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2533	}
2534
2535
2536	/**
2537	* Performs lazy restoration of the set of host MSRs if they were previously
2538	* loaded with guest MSR values.
2539	*
2540	* @param pVCpu The cross context virtual CPU structure.
2541	*
2542	* @remarks No-long-jump zone!!!
2543	* @remarks The guest MSRs should have been saved back into the guest-CPU
2544	* context by hmR0VmxImportGuestState()!!!
2545	*/
2546	static void hmR0VmxLazyRestoreHostMsrs(PVMCPUCC pVCpu)
2547	{
2548	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2549	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2550
2551	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2552	{
2553	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2554	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2555	{
2556	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2557	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2558	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2559	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2560	}
2561	}
2562	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2563	}
2564
2565
2566	/**
2567	* Verifies that our cached values of the VMCS fields are all consistent with
2568	* what's actually present in the VMCS.
2569	*
2570	* @returns VBox status code.
2571	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2572	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2573	* VMCS content. HMCPU error-field is
2574	* updated, see VMX_VCI_XXX.
2575	* @param pVCpu The cross context virtual CPU structure.
2576	* @param pVmcsInfo The VMCS info. object.
2577	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2578	*/
2579	static int hmR0VmxCheckVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2580	{
2581	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2582
2583	uint32_t u32Val;
2584	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2585	AssertRC(rc);
2586	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2587	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2588	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2589	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2590
2591	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2592	AssertRC(rc);
2593	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2594	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2595	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2596	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2597
2598	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2599	AssertRC(rc);
2600	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2601	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2602	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2603	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2604
2605	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2606	AssertRC(rc);
2607	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2608	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2609	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2610	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2611
2612	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2613	{
2614	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2615	AssertRC(rc);
2616	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2617	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2618	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2619	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2620	}
2621
2622	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2623	AssertRC(rc);
2624	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2625	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2626	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2627	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2628
2629	uint64_t u64Val;
2630	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2631	AssertRC(rc);
2632	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2633	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2634	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2635	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2636
2637	NOREF(pcszVmcs);
2638	return VINF_SUCCESS;
2639	}
2640
2641
2642	#ifdef VBOX_STRICT
2643	/**
2644	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2645	*
2646	* @param pVCpu The cross context virtual CPU structure.
2647	* @param pVmcsInfo The VMCS info. object.
2648	*/
2649	static void hmR0VmxCheckHostEferMsr(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2650	{
2651	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2652
2653	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2654	{
2655	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2656	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2657	uint64_t uVmcsEferMsrVmcs;
2658	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2659	AssertRC(rc);
2660
2661	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2662	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2663	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2664	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2665	}
2666	}
2667
2668
2669	/**
2670	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2671	* VMCS are correct.
2672	*
2673	* @param pVCpu The cross context virtual CPU structure.
2674	* @param pVmcsInfo The VMCS info. object.
2675	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2676	*/
2677	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2678	{
2679	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2680
2681	/* Read the various MSR-area counts from the VMCS. */
2682	uint32_t cEntryLoadMsrs;
2683	uint32_t cExitStoreMsrs;
2684	uint32_t cExitLoadMsrs;
2685	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2686	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2687	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2688
2689	/* Verify all the MSR counts are the same. */
2690	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2691	Assert(cExitStoreMsrs == cExitLoadMsrs);
2692	uint32_t const cMsrs = cExitLoadMsrs;
2693
2694	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2695	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2696
2697	/* Verify the MSR counts are within the allocated page size. */
2698	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2699
2700	/* Verify the relevant contents of the MSR areas match. */
2701	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2702	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2703	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2704	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2705	for (uint32_t i = 0; i < cMsrs; i++)
2706	{
2707	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2708	if (fSeparateExitMsrStorePage)
2709	{
2710	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2711	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2712	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2713	}
2714
2715	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2716	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2717	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2718
2719	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2720	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2721	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2722	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2723
2724	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2725	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2726	if (fIsEferMsr)
2727	{
2728	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2729	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2730	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2731	}
2732
2733	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2734	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2735	{
2736	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2737	if (fIsEferMsr)
2738	{
2739	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2740	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2741	}
2742	else
2743	{
2744	if (!fIsNstGstVmcs)
2745	{
2746	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2747	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2748	}
2749	else
2750	{
2751	/*
2752	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2753	* execute a nested-guest with MSR passthrough.
2754	*
2755	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2756	* allow passthrough too.
2757	*/
2758	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2759	Assert(pvMsrBitmapNstGst);
2760	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2761	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2762	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2763	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2764	}
2765	}
2766	}
2767
2768	/* Move to the next MSR. */
2769	pHostMsrLoad++;
2770	pGuestMsrLoad++;
2771	pGuestMsrStore++;
2772	}
2773	}
2774	#endif /* VBOX_STRICT */
2775
2776
2777	/**
2778	* Flushes the TLB using EPT.
2779	*
2780	* @returns VBox status code.
2781	* @param pVCpu The cross context virtual CPU structure of the calling
2782	* EMT. Can be NULL depending on @a enmTlbFlush.
2783	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2784	* enmTlbFlush.
2785	* @param enmTlbFlush Type of flush.
2786	*
2787	* @remarks Caller is responsible for making sure this function is called only
2788	* when NestedPaging is supported and providing @a enmTlbFlush that is
2789	* supported by the CPU.
2790	* @remarks Can be called with interrupts disabled.
2791	*/
2792	static void hmR0VmxFlushEpt(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2793	{
2794	uint64_t au64Descriptor[2];
2795	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2796	au64Descriptor[0] = 0;
2797	else
2798	{
2799	Assert(pVCpu);
2800	Assert(pVmcsInfo);
2801	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2802	}
2803	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2804
2805	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2806	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2807
2808	if ( RT_SUCCESS(rc)
2809	&& pVCpu)
2810	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2811	}
2812
2813
2814	/**
2815	* Flushes the TLB using VPID.
2816	*
2817	* @returns VBox status code.
2818	* @param pVCpu The cross context virtual CPU structure of the calling
2819	* EMT. Can be NULL depending on @a enmTlbFlush.
2820	* @param enmTlbFlush Type of flush.
2821	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2822	* on @a enmTlbFlush).
2823	*
2824	* @remarks Can be called with interrupts disabled.
2825	*/
2826	static void hmR0VmxFlushVpid(PVMCPUCC pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2827	{
2828	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2829
2830	uint64_t au64Descriptor[2];
2831	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2832	{
2833	au64Descriptor[0] = 0;
2834	au64Descriptor[1] = 0;
2835	}
2836	else
2837	{
2838	AssertPtr(pVCpu);
2839	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2840	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2841	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2842	au64Descriptor[1] = GCPtr;
2843	}
2844
2845	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2846	AssertMsg(rc == VINF_SUCCESS,
2847	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2848
2849	if ( RT_SUCCESS(rc)
2850	&& pVCpu)
2851	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2852	NOREF(rc);
2853	}
2854
2855
2856	/**
2857	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2858	* otherwise there is nothing really to invalidate.
2859	*
2860	* @returns VBox status code.
2861	* @param pVCpu The cross context virtual CPU structure.
2862	* @param GCVirt Guest virtual address of the page to invalidate.
2863	*/
2864	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
2865	{
2866	AssertPtr(pVCpu);
2867	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2868
2869	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2870	{
2871	/*
2872	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2873	* the EPT case. See @bugref{6043} and @bugref{6177}.
2874	*
2875	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2876	* as this function maybe called in a loop with individual addresses.
2877	*/
2878	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2879	if (pVM->hm.s.vmx.fVpid)
2880	{
2881	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2882	if (fVpidFlush)
2883	{
2884	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2885	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2886	}
2887	else
2888	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2889	}
2890	else if (pVM->hm.s.fNestedPaging)
2891	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2892	}
2893
2894	return VINF_SUCCESS;
2895	}
2896
2897
2898	/**
2899	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2900	* case where neither EPT nor VPID is supported by the CPU.
2901	*
2902	* @param pHostCpu The HM physical-CPU structure.
2903	* @param pVCpu The cross context virtual CPU structure.
2904	*
2905	* @remarks Called with interrupts disabled.
2906	*/
2907	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2908	{
2909	AssertPtr(pVCpu);
2910	AssertPtr(pHostCpu);
2911
2912	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2913
2914	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2915	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2916	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2917	pVCpu->hm.s.fForceTLBFlush = false;
2918	return;
2919	}
2920
2921
2922	/**
2923	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2924	*
2925	* @param pHostCpu The HM physical-CPU structure.
2926	* @param pVCpu The cross context virtual CPU structure.
2927	* @param pVmcsInfo The VMCS info. object.
2928	*
2929	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2930	* nomenclature. The reason is, to avoid confusion in compare statements
2931	* since the host-CPU copies are named "ASID".
2932	*
2933	* @remarks Called with interrupts disabled.
2934	*/
2935	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2936	{
2937	#ifdef VBOX_WITH_STATISTICS
2938	bool fTlbFlushed = false;
2939	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2940	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2941	if (!fTlbFlushed) \
2942	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2943	} while (0)
2944	#else
2945	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2946	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2947	#endif
2948
2949	AssertPtr(pVCpu);
2950	AssertPtr(pHostCpu);
2951	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2952
2953	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2954	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2955	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2956	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2957
2958	/*
2959	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2960	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2961	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2962	* cannot reuse the current ASID anymore.
2963	*/
2964	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2965	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2966	{
2967	++pHostCpu->uCurrentAsid;
2968	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2969	{
2970	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2971	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2972	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2973	}
2974
2975	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2976	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2977	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2978
2979	/*
2980	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2981	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2982	*/
2983	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2984	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2985	HMVMX_SET_TAGGED_TLB_FLUSHED();
2986	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2987	}
2988	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2989	{
2990	/*
2991	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2992	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2993	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2994	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2995	* mappings, see @bugref{6568}.
2996	*
2997	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2998	*/
2999	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3000	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3001	HMVMX_SET_TAGGED_TLB_FLUSHED();
3002	}
3003	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3004	{
3005	/*
3006	* The nested-guest specifies its own guest-physical address to use as the APIC-access
3007	* address which requires flushing the TLB of EPT cached structures.
3008	*
3009	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
3010	*/
3011	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3012	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3013	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3014	HMVMX_SET_TAGGED_TLB_FLUSHED();
3015	}
3016
3017
3018	pVCpu->hm.s.fForceTLBFlush = false;
3019	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3020
3021	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3022	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3023	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3024	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3025	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3026	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3027	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3028	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3029	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3030
3031	/* Update VMCS with the VPID. */
3032	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3033	AssertRC(rc);
3034
3035	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3036	}
3037
3038
3039	/**
3040	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3041	*
3042	* @param pHostCpu The HM physical-CPU structure.
3043	* @param pVCpu The cross context virtual CPU structure.
3044	* @param pVmcsInfo The VMCS info. object.
3045	*
3046	* @remarks Called with interrupts disabled.
3047	*/
3048	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3049	{
3050	AssertPtr(pVCpu);
3051	AssertPtr(pHostCpu);
3052	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3053	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3054	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3055
3056	/*
3057	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3058	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3059	*/
3060	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3061	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3062	{
3063	pVCpu->hm.s.fForceTLBFlush = true;
3064	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3065	}
3066
3067	/* Check for explicit TLB flushes. */
3068	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3069	{
3070	pVCpu->hm.s.fForceTLBFlush = true;
3071	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3072	}
3073
3074	/* Check for TLB flushes while switching to/from a nested-guest. */
3075	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3076	{
3077	pVCpu->hm.s.fForceTLBFlush = true;
3078	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3079	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3080	}
3081
3082	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3083	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3084
3085	if (pVCpu->hm.s.fForceTLBFlush)
3086	{
3087	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3088	pVCpu->hm.s.fForceTLBFlush = false;
3089	}
3090	}
3091
3092
3093	/**
3094	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3095	*
3096	* @param pHostCpu The HM physical-CPU structure.
3097	* @param pVCpu The cross context virtual CPU structure.
3098	*
3099	* @remarks Called with interrupts disabled.
3100	*/
3101	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
3102	{
3103	AssertPtr(pVCpu);
3104	AssertPtr(pHostCpu);
3105	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3106	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3107	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3108
3109	/*
3110	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3111	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3112	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3113	* cannot reuse the current ASID anymore.
3114	*/
3115	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3116	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3117	{
3118	pVCpu->hm.s.fForceTLBFlush = true;
3119	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3120	}
3121
3122	/* Check for explicit TLB flushes. */
3123	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3124	{
3125	/*
3126	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3127	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3128	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3129	* include fExplicitFlush's too) - an obscure corner case.
3130	*/
3131	pVCpu->hm.s.fForceTLBFlush = true;
3132	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3133	}
3134
3135	/* Check for TLB flushes while switching to/from a nested-guest. */
3136	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3137	{
3138	pVCpu->hm.s.fForceTLBFlush = true;
3139	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3140	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3141	}
3142
3143	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3144	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3145	if (pVCpu->hm.s.fForceTLBFlush)
3146	{
3147	++pHostCpu->uCurrentAsid;
3148	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3149	{
3150	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3151	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3152	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3153	}
3154
3155	pVCpu->hm.s.fForceTLBFlush = false;
3156	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3157	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3158	if (pHostCpu->fFlushAsidBeforeUse)
3159	{
3160	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3161	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3162	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3163	{
3164	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3165	pHostCpu->fFlushAsidBeforeUse = false;
3166	}
3167	else
3168	{
3169	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3170	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3171	}
3172	}
3173	}
3174
3175	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3176	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3177	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3178	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3179	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3180	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3181	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3182
3183	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3184	AssertRC(rc);
3185	}
3186
3187
3188	/**
3189	* Flushes the guest TLB entry based on CPU capabilities.
3190	*
3191	* @param pHostCpu The HM physical-CPU structure.
3192	* @param pVCpu The cross context virtual CPU structure.
3193	* @param pVmcsInfo The VMCS info. object.
3194	*
3195	* @remarks Called with interrupts disabled.
3196	*/
3197	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3198	{
3199	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3200	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3201	#endif
3202	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3203	switch (pVM->hm.s.vmx.enmTlbFlushType)
3204	{
3205	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3206	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3207	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3208	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3209	default:
3210	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3211	break;
3212	}
3213	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3214	}
3215
3216
3217	/**
3218	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3219	* TLB entries from the host TLB before VM-entry.
3220	*
3221	* @returns VBox status code.
3222	* @param pVM The cross context VM structure.
3223	*/
3224	static int hmR0VmxSetupTaggedTlb(PVMCC pVM)
3225	{
3226	/*
3227	* Determine optimal flush type for nested paging.
3228	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3229	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3230	*/
3231	if (pVM->hm.s.fNestedPaging)
3232	{
3233	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3234	{
3235	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3236	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3237	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3238	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3239	else
3240	{
3241	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3242	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3243	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3244	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3245	}
3246
3247	/* Make sure the write-back cacheable memory type for EPT is supported. */
3248	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3249	{
3250	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3251	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3252	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3253	}
3254
3255	/* EPT requires a page-walk length of 4. */
3256	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3257	{
3258	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3259	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3260	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3261	}
3262	}
3263	else
3264	{
3265	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3266	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3267	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3268	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3269	}
3270	}
3271
3272	/*
3273	* Determine optimal flush type for VPID.
3274	*/
3275	if (pVM->hm.s.vmx.fVpid)
3276	{
3277	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3278	{
3279	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3280	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3281	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3282	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3283	else
3284	{
3285	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3286	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3287	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3288	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3289	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3290	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3291	pVM->hm.s.vmx.fVpid = false;
3292	}
3293	}
3294	else
3295	{
3296	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3297	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3298	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3299	pVM->hm.s.vmx.fVpid = false;
3300	}
3301	}
3302
3303	/*
3304	* Setup the handler for flushing tagged-TLBs.
3305	*/
3306	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3307	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3308	else if (pVM->hm.s.fNestedPaging)
3309	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3310	else if (pVM->hm.s.vmx.fVpid)
3311	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3312	else
3313	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3314	return VINF_SUCCESS;
3315	}
3316
3317
3318	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3319	/**
3320	* Sets up the shadow VMCS fields arrays.
3321	*
3322	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3323	* executing the guest.
3324	*
3325	* @returns VBox status code.
3326	* @param pVM The cross context VM structure.
3327	*/
3328	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVMCC pVM)
3329	{
3330	/*
3331	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3332	* when the host does not support it.
3333	*/
3334	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3335	if ( !fGstVmwriteAll
3336	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3337	{ /* likely. */ }
3338	else
3339	{
3340	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3341	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3342	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3343	}
3344
3345	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3346	uint32_t cRwFields = 0;
3347	uint32_t cRoFields = 0;
3348	for (uint32_t i = 0; i < cVmcsFields; i++)
3349	{
3350	VMXVMCSFIELD VmcsField;
3351	VmcsField.u = g_aVmcsFields[i];
3352
3353	/*
3354	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3355	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3356	* in the shadow VMCS fields array as they would be redundant.
3357	*
3358	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3359	* we must not include it in the shadow VMCS fields array. Guests attempting to
3360	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3361	* the required behavior.
3362	*/
3363	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3364	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3365	{
3366	/*
3367	* Read-only fields are placed in a separate array so that while syncing shadow
3368	* VMCS fields later (which is more performance critical) we can avoid branches.
3369	*
3370	* However, if the guest can write to all fields (including read-only fields),
3371	* we treat it a as read/write field. Otherwise, writing to these fields would
3372	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3373	*/
3374	if ( fGstVmwriteAll
3375	\|\| !VMXIsVmcsFieldReadOnly(VmcsField.u))
3376	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3377	else
3378	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3379	}
3380	}
3381
3382	/* Update the counts. */
3383	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3384	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3385	return VINF_SUCCESS;
3386	}
3387
3388
3389	/**
3390	* Sets up the VMREAD and VMWRITE bitmaps.
3391	*
3392	* @param pVM The cross context VM structure.
3393	*/
3394	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVMCC pVM)
3395	{
3396	/*
3397	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3398	*/
3399	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3400	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3401	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3402	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3403	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3404
3405	/*
3406	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3407	* VMREAD and VMWRITE bitmaps.
3408	*/
3409	{
3410	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3411	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3412	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3413	{
3414	uint32_t const uVmcsField = paShadowVmcsFields[i];
3415	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3416	Assert(uVmcsField >> 3 < cbBitmap);
3417	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3418	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3419	}
3420	}
3421
3422	/*
3423	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3424	* if the host supports VMWRITE to all supported VMCS fields.
3425	*/
3426	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3427	{
3428	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3429	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3430	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3431	{
3432	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3433	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3434	Assert(uVmcsField >> 3 < cbBitmap);
3435	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3436	}
3437	}
3438	}
3439	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3440
3441
3442	/**
3443	* Sets up the virtual-APIC page address for the VMCS.
3444	*
3445	* @param pVmcsInfo The VMCS info. object.
3446	*/
3447	DECLINLINE(void) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
3448	{
3449	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3450	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3451	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3452	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3453	AssertRC(rc);
3454	}
3455
3456
3457	/**
3458	* Sets up the MSR-bitmap address for the VMCS.
3459	*
3460	* @param pVmcsInfo The VMCS info. object.
3461	*/
3462	DECLINLINE(void) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
3463	{
3464	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3465	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3466	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3467	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3468	AssertRC(rc);
3469	}
3470
3471
3472	/**
3473	* Sets up the APIC-access page address for the VMCS.
3474	*
3475	* @param pVCpu The cross context virtual CPU structure.
3476	*/
3477	DECLINLINE(void) hmR0VmxSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
3478	{
3479	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3480	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3481	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3482	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3483	AssertRC(rc);
3484	}
3485
3486
3487	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3488	/**
3489	* Sets up the VMREAD bitmap address for the VMCS.
3490	*
3491	* @param pVCpu The cross context virtual CPU structure.
3492	*/
3493	DECLINLINE(void) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
3494	{
3495	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3496	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3497	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3498	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3499	AssertRC(rc);
3500	}
3501
3502
3503	/**
3504	* Sets up the VMWRITE bitmap address for the VMCS.
3505	*
3506	* @param pVCpu The cross context virtual CPU structure.
3507	*/
3508	DECLINLINE(void) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
3509	{
3510	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3511	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3512	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3513	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3514	AssertRC(rc);
3515	}
3516	#endif
3517
3518
3519	/**
3520	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3521	* in the VMCS.
3522	*
3523	* @returns VBox status code.
3524	* @param pVmcsInfo The VMCS info. object.
3525	*/
3526	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
3527	{
3528	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3529	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3530	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3531
3532	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3533	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3534	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3535
3536	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3537	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3538	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3539
3540	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad); AssertRC(rc);
3541	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore); AssertRC(rc);
3542	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad); AssertRC(rc);
3543	return VINF_SUCCESS;
3544	}
3545
3546
3547	/**
3548	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3549	*
3550	* @param pVCpu The cross context virtual CPU structure.
3551	* @param pVmcsInfo The VMCS info. object.
3552	*/
3553	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3554	{
3555	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3556
3557	/*
3558	* The guest can access the following MSRs (read, write) without causing
3559	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3560	*/
3561	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3562	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3563	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3564	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3565	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3566	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3567
3568	/*
3569	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3570	* associated with then. We never need to intercept access (writes need to be
3571	* executed without causing a VM-exit, reads will #GP fault anyway).
3572	*
3573	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3574	* read/write them. We swap the the guest/host MSR value using the
3575	* auto-load/store MSR area.
3576	*/
3577	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3578	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3579	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3580	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3581	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3582	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3583
3584	/*
3585	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3586	* required for 64-bit guests.
3587	*/
3588	if (pVM->hm.s.fAllow64BitGuests)
3589	{
3590	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3591	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3592	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3593	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3594	}
3595
3596	/*
3597	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3598	*/
3599	#ifdef VBOX_STRICT
3600	Assert(pVmcsInfo->pvMsrBitmap);
3601	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3602	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3603	#endif
3604	}
3605
3606
3607	/**
3608	* Sets up pin-based VM-execution controls in the VMCS.
3609	*
3610	* @returns VBox status code.
3611	* @param pVCpu The cross context virtual CPU structure.
3612	* @param pVmcsInfo The VMCS info. object.
3613	*/
3614	static int hmR0VmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3615	{
3616	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3617	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3618	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3619
3620	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3621	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3622
3623	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3624	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3625
3626	/* Enable the VMX-preemption timer. */
3627	if (pVM->hm.s.vmx.fUsePreemptTimer)
3628	{
3629	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3630	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3631	}
3632
3633	#if 0
3634	/* Enable posted-interrupt processing. */
3635	if (pVM->hm.s.fPostedIntrs)
3636	{
3637	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3638	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3639	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3640	}
3641	#endif
3642
3643	if ((fVal & fZap) != fVal)
3644	{
3645	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3646	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3647	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3648	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3649	}
3650
3651	/* Commit it to the VMCS and update our cache. */
3652	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3653	AssertRC(rc);
3654	pVmcsInfo->u32PinCtls = fVal;
3655
3656	return VINF_SUCCESS;
3657	}
3658
3659
3660	/**
3661	* Sets up secondary processor-based VM-execution controls in the VMCS.
3662	*
3663	* @returns VBox status code.
3664	* @param pVCpu The cross context virtual CPU structure.
3665	* @param pVmcsInfo The VMCS info. object.
3666	*/
3667	static int hmR0VmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3668	{
3669	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3670	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3671	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3672
3673	/* WBINVD causes a VM-exit. */
3674	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3675	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3676
3677	/* Enable EPT (aka nested-paging). */
3678	if (pVM->hm.s.fNestedPaging)
3679	fVal \|= VMX_PROC_CTLS2_EPT;
3680
3681	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3682	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3683	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3684	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3685	fVal \|= VMX_PROC_CTLS2_INVPCID;
3686
3687	/* Enable VPID. */
3688	if (pVM->hm.s.vmx.fVpid)
3689	fVal \|= VMX_PROC_CTLS2_VPID;
3690
3691	/* Enable unrestricted guest execution. */
3692	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3693	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3694
3695	#if 0
3696	if (pVM->hm.s.fVirtApicRegs)
3697	{
3698	/* Enable APIC-register virtualization. */
3699	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3700	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3701
3702	/* Enable virtual-interrupt delivery. */
3703	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3704	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3705	}
3706	#endif
3707
3708	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3709	where the TPR shadow resides. */
3710	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3711	* done dynamically. */
3712	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3713	{
3714	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3715	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3716	}
3717
3718	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3719	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3720	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3721	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3722	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3723
3724	/* Enable Pause-Loop exiting. */
3725	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3726	&& pVM->hm.s.vmx.cPleGapTicks
3727	&& pVM->hm.s.vmx.cPleWindowTicks)
3728	{
3729	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3730
3731	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
3732	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
3733	}
3734
3735	if ((fVal & fZap) != fVal)
3736	{
3737	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3738	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3739	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3740	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3741	}
3742
3743	/* Commit it to the VMCS and update our cache. */
3744	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3745	AssertRC(rc);
3746	pVmcsInfo->u32ProcCtls2 = fVal;
3747
3748	return VINF_SUCCESS;
3749	}
3750
3751
3752	/**
3753	* Sets up processor-based VM-execution controls in the VMCS.
3754	*
3755	* @returns VBox status code.
3756	* @param pVCpu The cross context virtual CPU structure.
3757	* @param pVmcsInfo The VMCS info. object.
3758	*/
3759	static int hmR0VmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3760	{
3761	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3762
3763	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3764	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3765
3766	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3767	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3768	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3769	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3770	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3771	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3772	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3773
3774	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3775	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3776	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3777	{
3778	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3779	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3780	}
3781
3782	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3783	if (!pVM->hm.s.fNestedPaging)
3784	{
3785	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3786	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3787	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3788	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3789	}
3790
3791	/* Use TPR shadowing if supported by the CPU. */
3792	if ( PDMHasApic(pVM)
3793	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3794	{
3795	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3796	/* CR8 writes cause a VM-exit based on TPR threshold. */
3797	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3798	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3799	hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3800	}
3801	else
3802	{
3803	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3804	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3805	if (pVM->hm.s.fAllow64BitGuests)
3806	{
3807	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3808	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3809	}
3810	}
3811
3812	/* Use MSR-bitmaps if supported by the CPU. */
3813	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3814	{
3815	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3816	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3817	}
3818
3819	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3820	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3821	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3822
3823	if ((fVal & fZap) != fVal)
3824	{
3825	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3826	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3827	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3828	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3829	}
3830
3831	/* Commit it to the VMCS and update our cache. */
3832	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3833	AssertRC(rc);
3834	pVmcsInfo->u32ProcCtls = fVal;
3835
3836	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3837	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3838	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3839
3840	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3841	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3842	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3843
3844	/* Sanity check, should not really happen. */
3845	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3846	{ /* likely */ }
3847	else
3848	{
3849	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3850	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3851	}
3852
3853	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3854	return VINF_SUCCESS;
3855	}
3856
3857
3858	/**
3859	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3860	* Processor-based VM-execution) control fields in the VMCS.
3861	*
3862	* @returns VBox status code.
3863	* @param pVCpu The cross context virtual CPU structure.
3864	* @param pVmcsInfo The VMCS info. object.
3865	*/
3866	static int hmR0VmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3867	{
3868	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3869	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3870	{
3871	hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3872	hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3873	}
3874	#endif
3875
3876	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3877	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3878	AssertRC(rc);
3879
3880	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3881	if (RT_SUCCESS(rc))
3882	{
3883	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3884	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3885
3886	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
3887	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
3888
3889	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3890	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3891	return VINF_SUCCESS;
3892	}
3893	else
3894	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3895	return rc;
3896	}
3897
3898
3899	/**
3900	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3901	*
3902	* We shall setup those exception intercepts that don't change during the
3903	* lifetime of the VM here. The rest are done dynamically while loading the
3904	* guest state.
3905	*
3906	* @param pVCpu The cross context virtual CPU structure.
3907	* @param pVmcsInfo The VMCS info. object.
3908	*/
3909	static void hmR0VmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3910	{
3911	/*
3912	* The following exceptions are always intercepted:
3913	*
3914	* #AC - To prevent the guest from hanging the CPU.
3915	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3916	* recursive #DBs can cause a CPU hang.
3917	* #PF - To sync our shadow page tables when nested-paging is not used.
3918	*/
3919	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3920	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3921	\| RT_BIT(X86_XCPT_DB)
3922	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3923
3924	/* Commit it to the VMCS. */
3925	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3926	AssertRC(rc);
3927
3928	/* Update our cache of the exception bitmap. */
3929	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3930	}
3931
3932
3933	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3934	/**
3935	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3936	*
3937	* @returns VBox status code.
3938	* @param pVCpu The cross context virtual CPU structure.
3939	* @param pVmcsInfo The VMCS info. object.
3940	*/
3941	static int hmR0VmxSetupVmcsCtlsNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3942	{
3943	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3944	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3945	AssertRC(rc);
3946
3947	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3948	if (RT_SUCCESS(rc))
3949	{
3950	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3951	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3952
3953	/* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
3954	Assert(!pVmcsInfo->u64Cr0Mask);
3955	Assert(!pVmcsInfo->u64Cr4Mask);
3956	return VINF_SUCCESS;
3957	}
3958	else
3959	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3960	return rc;
3961	}
3962	#endif
3963
3964
3965	/**
3966	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3967	* VMX.
3968	*
3969	* @returns VBox status code.
3970	* @param pVCpu The cross context virtual CPU structure.
3971	* @param pVmcsInfo The VMCS info. object.
3972	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3973	*/
3974	static int hmR0VmxSetupVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3975	{
3976	Assert(pVmcsInfo->pvVmcs);
3977	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3978
3979	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3980	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3981	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3982	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3983
3984	LogFlowFunc(("\n"));
3985
3986	/*
3987	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3988	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3989	*/
3990	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3991	if (RT_SUCCESS(rc))
3992	{
3993	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3994	if (RT_SUCCESS(rc))
3995	{
3996	if (!fIsNstGstVmcs)
3997	{
3998	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3999	if (RT_SUCCESS(rc))
4000	{
4001	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4002	if (RT_SUCCESS(rc))
4003	{
4004	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4005	if (RT_SUCCESS(rc))
4006	{
4007	hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4008	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4009	/*
4010	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4011	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4012	* making it fit for use when VMCS shadowing is later enabled.
4013	*/
4014	if (pVmcsInfo->pvShadowVmcs)
4015	{
4016	VMXVMCSREVID VmcsRevId;
4017	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4018	VmcsRevId.n.fIsShadowVmcs = 1;
4019	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4020	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4021	if (RT_SUCCESS(rc))
4022	{ /* likely */ }
4023	else
4024	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4025	}
4026	#endif
4027	}
4028	else
4029	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4030	}
4031	else
4032	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4033	}
4034	else
4035	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4036	}
4037	else
4038	{
4039	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4040	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4041	if (RT_SUCCESS(rc))
4042	{ /* likely */ }
4043	else
4044	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4045	#else
4046	AssertFailed();
4047	#endif
4048	}
4049	}
4050	else
4051	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4052	}
4053	else
4054	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4055
4056	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4057	if (RT_SUCCESS(rc))
4058	{
4059	rc = hmR0VmxClearVmcs(pVmcsInfo);
4060	if (RT_SUCCESS(rc))
4061	{ /* likely */ }
4062	else
4063	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4064	}
4065
4066	/*
4067	* Update the last-error record both for failures and success, so we
4068	* can propagate the status code back to ring-3 for diagnostics.
4069	*/
4070	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4071	NOREF(pszVmcs);
4072	return rc;
4073	}
4074
4075
4076	/**
4077	* Does global VT-x initialization (called during module initialization).
4078	*
4079	* @returns VBox status code.
4080	*/
4081	VMMR0DECL(int) VMXR0GlobalInit(void)
4082	{
4083	#ifdef HMVMX_USE_FUNCTION_TABLE
4084	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4085	# ifdef VBOX_STRICT
4086	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4087	Assert(g_apfnVMExitHandlers[i]);
4088	# endif
4089	#endif
4090	return VINF_SUCCESS;
4091	}
4092
4093
4094	/**
4095	* Does global VT-x termination (called during module termination).
4096	*/
4097	VMMR0DECL(void) VMXR0GlobalTerm()
4098	{
4099	/* Nothing to do currently. */
4100	}
4101
4102
4103	/**
4104	* Sets up and activates VT-x on the current CPU.
4105	*
4106	* @returns VBox status code.
4107	* @param pHostCpu The HM physical-CPU structure.
4108	* @param pVM The cross context VM structure. Can be
4109	* NULL after a host resume operation.
4110	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4111	* fEnabledByHost is @c true).
4112	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4113	* @a fEnabledByHost is @c true).
4114	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4115	* enable VT-x on the host.
4116	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4117	*/
4118	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4119	PCSUPHWVIRTMSRS pHwvirtMsrs)
4120	{
4121	AssertPtr(pHostCpu);
4122	AssertPtr(pHwvirtMsrs);
4123	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4124
4125	/* Enable VT-x if it's not already enabled by the host. */
4126	if (!fEnabledByHost)
4127	{
4128	int rc = hmR0VmxEnterRootMode(pHostCpu, pVM, HCPhysCpuPage, pvCpuPage);
4129	if (RT_FAILURE(rc))
4130	return rc;
4131	}
4132
4133	/*
4134	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4135	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4136	* invalidated when flushing by VPID.
4137	*/
4138	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4139	{
4140	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4141	pHostCpu->fFlushAsidBeforeUse = false;
4142	}
4143	else
4144	pHostCpu->fFlushAsidBeforeUse = true;
4145
4146	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4147	++pHostCpu->cTlbFlushes;
4148
4149	return VINF_SUCCESS;
4150	}
4151
4152
4153	/**
4154	* Deactivates VT-x on the current CPU.
4155	*
4156	* @returns VBox status code.
4157	* @param pHostCpu The HM physical-CPU structure.
4158	* @param pvCpuPage Pointer to the VMXON region.
4159	* @param HCPhysCpuPage Physical address of the VMXON region.
4160	*
4161	* @remarks This function should never be called when SUPR0EnableVTx() or
4162	* similar was used to enable VT-x on the host.
4163	*/
4164	VMMR0DECL(int) VMXR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4165	{
4166	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4167
4168	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4169	return hmR0VmxLeaveRootMode(pHostCpu);
4170	}
4171
4172
4173	/**
4174	* Does per-VM VT-x initialization.
4175	*
4176	* @returns VBox status code.
4177	* @param pVM The cross context VM structure.
4178	*/
4179	VMMR0DECL(int) VMXR0InitVM(PVMCC pVM)
4180	{
4181	AssertPtr(pVM);
4182	LogFlowFunc(("pVM=%p\n", pVM));
4183
4184	int rc = hmR0VmxStructsAlloc(pVM);
4185	if (RT_FAILURE(rc))
4186	{
4187	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4188	return rc;
4189	}
4190
4191	return VINF_SUCCESS;
4192	}
4193
4194
4195	/**
4196	* Does per-VM VT-x termination.
4197	*
4198	* @returns VBox status code.
4199	* @param pVM The cross context VM structure.
4200	*/
4201	VMMR0DECL(int) VMXR0TermVM(PVMCC pVM)
4202	{
4203	AssertPtr(pVM);
4204	LogFlowFunc(("pVM=%p\n", pVM));
4205
4206	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4207	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4208	{
4209	Assert(pVM->hm.s.vmx.pvScratch);
4210	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4211	}
4212	#endif
4213	hmR0VmxStructsFree(pVM);
4214	return VINF_SUCCESS;
4215	}
4216
4217
4218	/**
4219	* Sets up the VM for execution using hardware-assisted VMX.
4220	* This function is only called once per-VM during initialization.
4221	*
4222	* @returns VBox status code.
4223	* @param pVM The cross context VM structure.
4224	*/
4225	VMMR0DECL(int) VMXR0SetupVM(PVMCC pVM)
4226	{
4227	AssertPtr(pVM);
4228	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4229
4230	LogFlowFunc(("pVM=%p\n", pVM));
4231
4232	/*
4233	* At least verify if VMX is enabled, since we can't check if we're in VMX root mode or not
4234	* without causing a #GP.
4235	*/
4236	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4237	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4238	{ /* likely */ }
4239	else
4240	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4241
4242	/*
4243	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4244	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4245	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4246	*/
4247	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4248	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4249	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4250	{
4251	LogRelFunc(("Invalid real-on-v86 state.\n"));
4252	return VERR_INTERNAL_ERROR;
4253	}
4254
4255	/* Initialize these always, see hmR3InitFinalizeR0().*/
4256	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4257	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4258
4259	/* Setup the tagged-TLB flush handlers. */
4260	int rc = hmR0VmxSetupTaggedTlb(pVM);
4261	if (RT_FAILURE(rc))
4262	{
4263	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4264	return rc;
4265	}
4266
4267	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4268	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4269	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4270	{
4271	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4272	if (RT_SUCCESS(rc))
4273	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4274	else
4275	{
4276	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4277	return rc;
4278	}
4279	}
4280	#endif
4281
4282	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4283	{
4284	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
4285	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4286
4287	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4288	if (RT_SUCCESS(rc))
4289	{
4290	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4291	if (pVM->cpum.ro.GuestFeatures.fVmx)
4292	{
4293	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4294	if (RT_SUCCESS(rc))
4295	{ /* likely */ }
4296	else
4297	{
4298	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4299	return rc;
4300	}
4301	}
4302	#endif
4303	}
4304	else
4305	{
4306	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4307	return rc;
4308	}
4309	}
4310
4311	return VINF_SUCCESS;
4312	}
4313
4314
4315	/**
4316	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4317	* the VMCS.
4318	*/
4319	static void hmR0VmxExportHostControlRegs(void)
4320	{
4321	int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, ASMGetCR0()); AssertRC(rc);
4322	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, ASMGetCR3()); AssertRC(rc);
4323	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, ASMGetCR4()); AssertRC(rc);
4324	}
4325
4326
4327	/**
4328	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4329	* the host-state area in the VMCS.
4330	*
4331	* @returns VBox status code.
4332	* @param pVCpu The cross context virtual CPU structure.
4333	*/
4334	static int hmR0VmxExportHostSegmentRegs(PVMCPUCC pVCpu)
4335	{
4336	/**
4337	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4338	* requirements. See hmR0VmxExportHostSegmentRegs().
4339	*/
4340	#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_selValue) \
4341	if ((a_selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4342	{ \
4343	bool fValidSelector = true; \
4344	if ((a_selValue) & X86_SEL_LDT) \
4345	{ \
4346	uint32_t const uAttr = ASMGetSegAttr(a_selValue); \
4347	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4348	} \
4349	if (fValidSelector) \
4350	{ \
4351	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##a_Seg; \
4352	pVCpu->hm.s.vmx.RestoreHost.uHostSel##a_Seg = (a_selValue); \
4353	} \
4354	(a_selValue) = 0; \
4355	}
4356
4357	/*
4358	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4359	* will be messed up. We should -not- save the messed up state without restoring
4360	* the original host-state, see @bugref{7240}.
4361	*
4362	* This apparently can happen (most likely the FPU changes), deal with it rather than
4363	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4364	*/
4365	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4366	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4367	{
4368	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4369	pVCpu->idCpu));
4370	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4371	}
4372	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4373
4374	/*
4375	* Host segment registers.
4376	*/
4377	RTSEL uSelES = ASMGetES();
4378	RTSEL uSelCS = ASMGetCS();
4379	RTSEL uSelSS = ASMGetSS();
4380	RTSEL uSelDS = ASMGetDS();
4381	RTSEL uSelFS = ASMGetFS();
4382	RTSEL uSelGS = ASMGetGS();
4383	RTSEL uSelTR = ASMGetTR();
4384
4385	/*
4386	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4387	* gain VM-entry and restore them before we get preempted.
4388	*
4389	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4390	*/
4391	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4392	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4393	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4394	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4395
4396	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4397	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4398	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4399	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4400	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4401	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4402	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4403	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4404	Assert(uSelCS);
4405	Assert(uSelTR);
4406
4407	/* Write these host selector fields into the host-state area in the VMCS. */
4408	int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS); AssertRC(rc);
4409	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS); AssertRC(rc);
4410	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS); AssertRC(rc);
4411	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES); AssertRC(rc);
4412	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS); AssertRC(rc);
4413	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS); AssertRC(rc);
4414	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR); AssertRC(rc);
4415
4416	/*
4417	* Host GDTR and IDTR.
4418	*/
4419	RTGDTR Gdtr;
4420	RTIDTR Idtr;
4421	RT_ZERO(Gdtr);
4422	RT_ZERO(Idtr);
4423	ASMGetGDTR(&Gdtr);
4424	ASMGetIDTR(&Idtr);
4425	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRC(rc);
4426	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRC(rc);
4427
4428	/*
4429	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4430	* them to the maximum limit (0xffff) on every VM-exit.
4431	*/
4432	if (Gdtr.cbGdt != 0xffff)
4433	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4434
4435	/*
4436	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4437	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4438	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4439	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4440	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4441	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4442	* at 0xffff on hosts where we are sure it won't cause trouble.
4443	*/
4444	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4445	if (Idtr.cbIdt < 0x0fff)
4446	#else
4447	if (Idtr.cbIdt != 0xffff)
4448	#endif
4449	{
4450	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4451	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4452	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4453	}
4454
4455	/*
4456	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4457	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4458	* RPL should be too in most cases.
4459	*/
4460	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4461	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4462
4463	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4464	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4465
4466	/*
4467	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4468	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4469	* restoration if the host has something else. Task switching is not supported in 64-bit
4470	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4471	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4472	*
4473	* [1] See Intel spec. 3.5 "System Descriptor Types".
4474	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4475	*/
4476	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4477	Assert(pDesc->System.u4Type == 11);
4478	if ( pDesc->System.u16LimitLow != 0x67
4479	\|\| pDesc->System.u4LimitHigh)
4480	{
4481	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4482	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4483	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4484	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4485	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4486	}
4487
4488	/*
4489	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4490	*/
4491	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4492	{
4493	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4494	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4495	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4496	{
4497	/* The GDT is read-only but the writable GDT is available. */
4498	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4499	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4500	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4501	AssertRCReturn(rc, rc);
4502	}
4503	}
4504
4505	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase);
4506	AssertRC(rc);
4507
4508	/*
4509	* Host FS base and GS base.
4510	*/
4511	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4512	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4513	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRC(rc);
4514	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRC(rc);
4515
4516	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4517	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4518	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4519	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4520	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4521
4522	return VINF_SUCCESS;
4523	#undef VMXLOCAL_ADJUST_HOST_SEG
4524	}
4525
4526
4527	/**
4528	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4529	* host-state area of the VMCS.
4530	*
4531	* These MSRs will be automatically restored on the host after every successful
4532	* VM-exit.
4533	*
4534	* @param pVCpu The cross context virtual CPU structure.
4535	*
4536	* @remarks No-long-jump zone!!!
4537	*/
4538	static void hmR0VmxExportHostMsrs(PVMCPUCC pVCpu)
4539	{
4540	AssertPtr(pVCpu);
4541
4542	/*
4543	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4544	* rather than swapping them on every VM-entry.
4545	*/
4546	hmR0VmxLazySaveHostMsrs(pVCpu);
4547
4548	/*
4549	* Host Sysenter MSRs.
4550	*/
4551	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)); AssertRC(rc);
4552	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP)); AssertRC(rc);
4553	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP)); AssertRC(rc);
4554
4555	/*
4556	* Host EFER MSR.
4557	*
4558	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4559	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4560	*/
4561	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4562	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4563	{
4564	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4565	AssertRC(rc);
4566	}
4567
4568	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4569	* hmR0VmxExportGuestEntryExitCtls(). */
4570	}
4571
4572
4573	/**
4574	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4575	*
4576	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4577	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4578	*
4579	* @returns true if we need to load guest EFER, false otherwise.
4580	* @param pVCpu The cross context virtual CPU structure.
4581	* @param pVmxTransient The VMX-transient structure.
4582	*
4583	* @remarks Requires EFER, CR4.
4584	* @remarks No-long-jump zone!!!
4585	*/
4586	static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4587	{
4588	#ifdef HMVMX_ALWAYS_SWAP_EFER
4589	RT_NOREF2(pVCpu, pVmxTransient);
4590	return true;
4591	#else
4592	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4593	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4594	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4595	uint64_t const u64GuestEfer = pCtx->msrEFER;
4596
4597	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4598	/*
4599	* For nested-guests, we shall honor swapping the EFER MSR when requested by
4600	* the nested-guest.
4601	*/
4602	if ( pVmxTransient->fIsNestedGuest
4603	&& ( CPUMIsGuestVmxEntryCtlsSet(pVCpu, pCtx, VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4604	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_SAVE_EFER_MSR)
4605	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_LOAD_EFER_MSR)))
4606	return true;
4607	# else
4608	RT_NOREF(pVmxTransient);
4609	#endif
4610
4611	/*
4612	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4613	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4614	*/
4615	if ( CPUMIsGuestInLongModeEx(pCtx)
4616	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4617	return true;
4618
4619	/*
4620	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4621	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4622	*
4623	* See Intel spec. 4.5 "IA-32e Paging".
4624	* See Intel spec. 4.1.1 "Three Paging Modes".
4625	*
4626	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4627	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4628	*/
4629	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4630	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4631	if ( (pCtx->cr4 & X86_CR4_PAE)
4632	&& (pCtx->cr0 & X86_CR0_PG))
4633	{
4634	/*
4635	* If nested paging is not used, verify that the guest paging mode matches the
4636	* shadow paging mode which is/will be placed in the VMCS (which is what will
4637	* actually be used while executing the guest and not the CR4 shadow value).
4638	*/
4639	AssertMsg(pVM->hm.s.fNestedPaging \|\| ( pVCpu->hm.s.enmShadowMode == PGMMODE_PAE
4640	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_PAE_NX
4641	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64
4642	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64_NX),
4643	("enmShadowMode=%u\n", pVCpu->hm.s.enmShadowMode));
4644	if ((u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4645	{
4646	/* Verify that the host is NX capable. */
4647	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4648	return true;
4649	}
4650	}
4651
4652	return false;
4653	#endif
4654	}
4655
4656
4657	/**
4658	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4659	* VMCS.
4660	*
4661	* This is typically required when the guest changes paging mode.
4662	*
4663	* @returns VBox status code.
4664	* @param pVCpu The cross context virtual CPU structure.
4665	* @param pVmxTransient The VMX-transient structure.
4666	*
4667	* @remarks Requires EFER.
4668	* @remarks No-long-jump zone!!!
4669	*/
4670	static int hmR0VmxExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4671	{
4672	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4673	{
4674	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4675	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4676	bool const fGstInLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
4677
4678	/*
4679	* VMRUN function.
4680	* If the guest is in long mode, use the 64-bit guest handler, else the 32-bit guest handler.
4681	* The host is always 64-bit since we no longer support 32-bit hosts.
4682	*/
4683	if (fGstInLongMode)
4684	{
4685	#ifndef VBOX_WITH_64_BITS_GUESTS
4686	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4687	#else
4688	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4689	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
4690	#endif
4691	}
4692	else
4693	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
4694
4695	/*
4696	* VM-entry controls.
4697	*/
4698	{
4699	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4700	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4701
4702	/*
4703	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4704	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4705	*
4706	* For nested-guests, this is a mandatory VM-entry control. It's also
4707	* required because we do not want to leak host bits to the nested-guest.
4708	*/
4709	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4710
4711	/*
4712	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4713	*
4714	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4715	* required to get the nested-guest working with hardware-assisted VMX execution.
4716	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
4717	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4718	* here rather than while merging the guest VMCS controls.
4719	*/
4720	if (fGstInLongMode)
4721	{
4722	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
4723	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4724	}
4725	else
4726	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4727
4728	/*
4729	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4730	*
4731	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4732	* regardless of whether the nested-guest VMCS specifies it because we are free to
4733	* load whatever MSRs we require and we do not need to modify the guest visible copy
4734	* of the VM-entry MSR load area.
4735	*/
4736	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4737	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4738	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4739	else
4740	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4741
4742	/*
4743	* The following should -not- be set (since we're not in SMM mode):
4744	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4745	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4746	*/
4747
4748	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4749	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4750
4751	if ((fVal & fZap) == fVal)
4752	{ /* likely */ }
4753	else
4754	{
4755	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4756	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4757	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4758	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4759	}
4760
4761	/* Commit it to the VMCS. */
4762	if (pVmcsInfo->u32EntryCtls != fVal)
4763	{
4764	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4765	AssertRC(rc);
4766	pVmcsInfo->u32EntryCtls = fVal;
4767	}
4768	}
4769
4770	/*
4771	* VM-exit controls.
4772	*/
4773	{
4774	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4775	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4776
4777	/*
4778	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4779	* supported the 1-setting of this bit.
4780	*
4781	* For nested-guests, we set the "save debug controls" as the converse
4782	* "load debug controls" is mandatory for nested-guests anyway.
4783	*/
4784	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4785
4786	/*
4787	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4788	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4789	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4790	* hmR0VmxExportHostMsrs().
4791	*
4792	* For nested-guests, we always set this bit as we do not support 32-bit
4793	* hosts.
4794	*/
4795	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4796
4797	/*
4798	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4799	*
4800	* For nested-guests, we should use the "save IA32_EFER" control if we also
4801	* used the "load IA32_EFER" control while exporting VM-entry controls.
4802	*/
4803	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4804	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4805	{
4806	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4807	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4808	}
4809
4810	/*
4811	* Enable saving of the VMX-preemption timer value on VM-exit.
4812	* For nested-guests, currently not exposed/used.
4813	*/
4814	if ( pVM->hm.s.vmx.fUsePreemptTimer
4815	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4816	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4817
4818	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4819	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4820
4821	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4822	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4823	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4824
4825	if ((fVal & fZap) == fVal)
4826	{ /* likely */ }
4827	else
4828	{
4829	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4830	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4831	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4832	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4833	}
4834
4835	/* Commit it to the VMCS. */
4836	if (pVmcsInfo->u32ExitCtls != fVal)
4837	{
4838	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4839	AssertRC(rc);
4840	pVmcsInfo->u32ExitCtls = fVal;
4841	}
4842	}
4843
4844	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4845	}
4846	return VINF_SUCCESS;
4847	}
4848
4849
4850	/**
4851	* Sets the TPR threshold in the VMCS.
4852	*
4853	* @param pVmcsInfo The VMCS info. object.
4854	* @param u32TprThreshold The TPR threshold (task-priority class only).
4855	*/
4856	DECLINLINE(void) hmR0VmxApicSetTprThreshold(PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4857	{
4858	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4859	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4860	RT_NOREF(pVmcsInfo);
4861	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4862	AssertRC(rc);
4863	}
4864
4865
4866	/**
4867	* Exports the guest APIC TPR state into the VMCS.
4868	*
4869	* @param pVCpu The cross context virtual CPU structure.
4870	* @param pVmxTransient The VMX-transient structure.
4871	*
4872	* @remarks No-long-jump zone!!!
4873	*/
4874	static void hmR0VmxExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4875	{
4876	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4877	{
4878	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4879
4880	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4881	if (!pVmxTransient->fIsNestedGuest)
4882	{
4883	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4884	&& APICIsEnabled(pVCpu))
4885	{
4886	/*
4887	* Setup TPR shadowing.
4888	*/
4889	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4890	{
4891	bool fPendingIntr = false;
4892	uint8_t u8Tpr = 0;
4893	uint8_t u8PendingIntr = 0;
4894	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4895	AssertRC(rc);
4896
4897	/*
4898	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4899	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4900	* priority of the pending interrupt so we can deliver the interrupt. If there
4901	* are no interrupts pending, set threshold to 0 to not cause any
4902	* TPR-below-threshold VM-exits.
4903	*/
4904	uint32_t u32TprThreshold = 0;
4905	if (fPendingIntr)
4906	{
4907	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4908	(which is the Task-Priority Class). */
4909	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4910	const uint8_t u8TprPriority = u8Tpr >> 4;
4911	if (u8PendingPriority <= u8TprPriority)
4912	u32TprThreshold = u8PendingPriority;
4913	}
4914
4915	hmR0VmxApicSetTprThreshold(pVmcsInfo, u32TprThreshold);
4916	}
4917	}
4918	}
4919	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4920	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4921	}
4922	}
4923
4924
4925	/**
4926	* Gets the guest interruptibility-state.
4927	*
4928	* @returns Guest's interruptibility-state.
4929	* @param pVCpu The cross context virtual CPU structure.
4930	* @param pVmxTransient The VMX-transient structure.
4931	*
4932	* @remarks No-long-jump zone!!!
4933	*/
4934	static uint32_t hmR0VmxGetGuestIntrState(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4935	{
4936	/*
4937	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4938	*/
4939	uint32_t fIntrState = 0;
4940	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4941	{
4942	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
4943	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
4944
4945	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4946	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4947	{
4948	if (pCtx->eflags.Bits.u1IF)
4949	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4950	else
4951	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4952	}
4953	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4954	{
4955	/*
4956	* We can clear the inhibit force flag as even if we go back to the recompiler
4957	* without executing guest code in VT-x, the flag's condition to be cleared is
4958	* met and thus the cleared state is correct.
4959	*/
4960	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4961	}
4962	}
4963
4964	/*
4965	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4966	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4967	* setting this would block host-NMIs and IRET will not clear the blocking.
4968	*
4969	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4970	*
4971	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4972	*/
4973	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4974	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4975	&& CPUMIsGuestNmiBlocking(pVCpu))
4976	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4977
4978	return fIntrState;
4979	}
4980
4981
4982	/**
4983	* Exports the exception intercepts required for guest execution in the VMCS.
4984	*
4985	* @param pVCpu The cross context virtual CPU structure.
4986	* @param pVmxTransient The VMX-transient structure.
4987	*
4988	* @remarks No-long-jump zone!!!
4989	*/
4990	static void hmR0VmxExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4991	{
4992	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4993	{
4994	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4995	if ( !pVmxTransient->fIsNestedGuest
4996	&& pVCpu->hm.s.fGIMTrapXcptUD)
4997	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4998	else
4999	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5000
5001	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5002	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5003	}
5004	}
5005
5006
5007	/**
5008	* Exports the guest's RIP into the guest-state area in the VMCS.
5009	*
5010	* @param pVCpu The cross context virtual CPU structure.
5011	*
5012	* @remarks No-long-jump zone!!!
5013	*/
5014	static void hmR0VmxExportGuestRip(PVMCPUCC pVCpu)
5015	{
5016	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5017	{
5018	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5019
5020	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5021	AssertRC(rc);
5022
5023	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5024	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5025	}
5026	}
5027
5028
5029	/**
5030	* Exports the guest's RSP into the guest-state area in the VMCS.
5031	*
5032	* @param pVCpu The cross context virtual CPU structure.
5033	*
5034	* @remarks No-long-jump zone!!!
5035	*/
5036	static void hmR0VmxExportGuestRsp(PVMCPUCC pVCpu)
5037	{
5038	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5039	{
5040	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5041
5042	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5043	AssertRC(rc);
5044
5045	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5046	Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
5047	}
5048	}
5049
5050
5051	/**
5052	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5053	*
5054	* @param pVCpu The cross context virtual CPU structure.
5055	* @param pVmxTransient The VMX-transient structure.
5056	*
5057	* @remarks No-long-jump zone!!!
5058	*/
5059	static void hmR0VmxExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5060	{
5061	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5062	{
5063	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5064
5065	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5066	Let us assert it as such and use 32-bit VMWRITE. */
5067	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5068	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5069	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5070	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5071
5072	/*
5073	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5074	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5075	* can run the real-mode guest code under Virtual 8086 mode.
5076	*/
5077	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5078	if (pVmcsInfo->RealMode.fRealOnV86Active)
5079	{
5080	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5081	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5082	Assert(!pVmxTransient->fIsNestedGuest);
5083	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5084	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5085	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5086	}
5087
5088	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5089	AssertRC(rc);
5090
5091	/*
5092	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5093	*
5094	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5095	* through the hypervisor debugger using EFLAGS.TF.
5096	*/
5097	if ( !pVmxTransient->fIsNestedGuest
5098	&& !pVCpu->hm.s.fSingleInstruction
5099	&& fEFlags.Bits.u1TF)
5100	{
5101	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5102	* premature trips to ring-3 esp since IEM does not yet handle it. */
5103	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5104	AssertRC(rc);
5105	}
5106	/* else: for nested-guest currently handling while merging controls. */
5107
5108	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5109	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5110	}
5111	}
5112
5113
5114	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5115	/**
5116	* Copies the nested-guest VMCS to the shadow VMCS.
5117	*
5118	* @returns VBox status code.
5119	* @param pVCpu The cross context virtual CPU structure.
5120	* @param pVmcsInfo The VMCS info. object.
5121	*
5122	* @remarks No-long-jump zone!!!
5123	*/
5124	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5125	{
5126	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5127	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5128
5129	/*
5130	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5131	* current VMCS, as we may try saving guest lazy MSRs.
5132	*
5133	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5134	* calling the import VMCS code which is currently performing the guest MSR reads
5135	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5136	* and the rest of the VMX leave session machinery.
5137	*/
5138	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5139
5140	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5141	if (RT_SUCCESS(rc))
5142	{
5143	/*
5144	* Copy all guest read/write VMCS fields.
5145	*
5146	* We don't check for VMWRITE failures here for performance reasons and
5147	* because they are not expected to fail, barring irrecoverable conditions
5148	* like hardware errors.
5149	*/
5150	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5151	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5152	{
5153	uint64_t u64Val;
5154	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5155	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5156	VMXWriteVmcs64(uVmcsField, u64Val);
5157	}
5158
5159	/*
5160	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5161	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5162	*/
5163	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5164	{
5165	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5166	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5167	{
5168	uint64_t u64Val;
5169	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5170	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5171	VMXWriteVmcs64(uVmcsField, u64Val);
5172	}
5173	}
5174
5175	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5176	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5177	}
5178
5179	ASMSetFlags(fEFlags);
5180	return rc;
5181	}
5182
5183
5184	/**
5185	* Copies the shadow VMCS to the nested-guest VMCS.
5186	*
5187	* @returns VBox status code.
5188	* @param pVCpu The cross context virtual CPU structure.
5189	* @param pVmcsInfo The VMCS info. object.
5190	*
5191	* @remarks Called with interrupts disabled.
5192	*/
5193	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5194	{
5195	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5196	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5197	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5198
5199	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5200	if (RT_SUCCESS(rc))
5201	{
5202	/*
5203	* Copy guest read/write fields from the shadow VMCS.
5204	* Guest read-only fields cannot be modified, so no need to copy them.
5205	*
5206	* We don't check for VMREAD failures here for performance reasons and
5207	* because they are not expected to fail, barring irrecoverable conditions
5208	* like hardware errors.
5209	*/
5210	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5211	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5212	{
5213	uint64_t u64Val;
5214	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5215	VMXReadVmcs64(uVmcsField, &u64Val);
5216	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5217	}
5218
5219	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5220	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5221	}
5222	return rc;
5223	}
5224
5225
5226	/**
5227	* Enables VMCS shadowing for the given VMCS info. object.
5228	*
5229	* @param pVmcsInfo The VMCS info. object.
5230	*
5231	* @remarks No-long-jump zone!!!
5232	*/
5233	static void hmR0VmxEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5234	{
5235	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5236	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5237	{
5238	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5239	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5240	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5241	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
5242	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5243	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5244	Log4Func(("Enabled\n"));
5245	}
5246	}
5247
5248
5249	/**
5250	* Disables VMCS shadowing for the given VMCS info. object.
5251	*
5252	* @param pVmcsInfo The VMCS info. object.
5253	*
5254	* @remarks No-long-jump zone!!!
5255	*/
5256	static void hmR0VmxDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5257	{
5258	/*
5259	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5260	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5261	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5262	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5263	*
5264	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5265	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5266	*/
5267	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5268	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5269	{
5270	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5271	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5272	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
5273	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5274	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5275	Log4Func(("Disabled\n"));
5276	}
5277	}
5278	#endif
5279
5280
5281	/**
5282	* Exports the guest hardware-virtualization state.
5283	*
5284	* @returns VBox status code.
5285	* @param pVCpu The cross context virtual CPU structure.
5286	* @param pVmxTransient The VMX-transient structure.
5287	*
5288	* @remarks No-long-jump zone!!!
5289	*/
5290	static int hmR0VmxExportGuestHwvirtState(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5291	{
5292	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5293	{
5294	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5295	/*
5296	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5297	* VMCS shadowing.
5298	*/
5299	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5300	{
5301	/*
5302	* If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
5303	* copy the nested hypervisor's current VMCS into the shadow VMCS and enable
5304	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5305	*
5306	* We check for VMX root mode here in case the guest executes VMXOFF without
5307	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5308	* not clear the current VMCS pointer.
5309	*/
5310	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5311	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5312	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5313	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5314	{
5315	/* Paranoia. */
5316	Assert(!pVmxTransient->fIsNestedGuest);
5317
5318	/*
5319	* For performance reasons, also check if the nested hypervisor's current VMCS
5320	* was newly loaded or modified before copying it to the shadow VMCS.
5321	*/
5322	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5323	{
5324	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5325	AssertRCReturn(rc, rc);
5326	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5327	}
5328	hmR0VmxEnableVmcsShadowing(pVmcsInfo);
5329	}
5330	else
5331	hmR0VmxDisableVmcsShadowing(pVmcsInfo);
5332	}
5333	#else
5334	NOREF(pVmxTransient);
5335	#endif
5336	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5337	}
5338	return VINF_SUCCESS;
5339	}
5340
5341
5342	/**
5343	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5344	*
5345	* The guest FPU state is always pre-loaded hence we don't need to bother about
5346	* sharing FPU related CR0 bits between the guest and host.
5347	*
5348	* @returns VBox status code.
5349	* @param pVCpu The cross context virtual CPU structure.
5350	* @param pVmxTransient The VMX-transient structure.
5351	*
5352	* @remarks No-long-jump zone!!!
5353	*/
5354	static int hmR0VmxExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5355	{
5356	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5357	{
5358	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5359	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5360
5361	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0;
5362	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5363	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5364	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5365	else
5366	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5367
5368	if (!pVmxTransient->fIsNestedGuest)
5369	{
5370	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5371	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5372	uint64_t const u64ShadowCr0 = u64GuestCr0;
5373	Assert(!RT_HI_U32(u64GuestCr0));
5374
5375	/*
5376	* Setup VT-x's view of the guest CR0.
5377	*/
5378	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5379	if (pVM->hm.s.fNestedPaging)
5380	{
5381	if (CPUMIsGuestPagingEnabled(pVCpu))
5382	{
5383	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5384	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5385	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5386	}
5387	else
5388	{
5389	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5390	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5391	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5392	}
5393
5394	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5395	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5396	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5397	}
5398	else
5399	{
5400	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5401	u64GuestCr0 \|= X86_CR0_WP;
5402	}
5403
5404	/*
5405	* Guest FPU bits.
5406	*
5407	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5408	* using CR0.TS.
5409	*
5410	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5411	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5412	*/
5413	u64GuestCr0 \|= X86_CR0_NE;
5414
5415	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5416	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5417
5418	/*
5419	* Update exception intercepts.
5420	*/
5421	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5422	if (pVmcsInfo->RealMode.fRealOnV86Active)
5423	{
5424	Assert(PDMVmmDevHeapIsEnabled(pVM));
5425	Assert(pVM->hm.s.vmx.pRealModeTSS);
5426	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5427	}
5428	else
5429	{
5430	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5431	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5432	if (fInterceptMF)
5433	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5434	}
5435
5436	/* Additional intercepts for debugging, define these yourself explicitly. */
5437	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5438	uXcptBitmap \|= 0
5439	\| RT_BIT(X86_XCPT_BP)
5440	\| RT_BIT(X86_XCPT_DE)
5441	\| RT_BIT(X86_XCPT_NM)
5442	\| RT_BIT(X86_XCPT_TS)
5443	\| RT_BIT(X86_XCPT_UD)
5444	\| RT_BIT(X86_XCPT_NP)
5445	\| RT_BIT(X86_XCPT_SS)
5446	\| RT_BIT(X86_XCPT_GP)
5447	\| RT_BIT(X86_XCPT_PF)
5448	\| RT_BIT(X86_XCPT_MF)
5449	;
5450	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5451	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5452	#endif
5453	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5454	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5455	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5456
5457	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5458	u64GuestCr0 \|= fSetCr0;
5459	u64GuestCr0 &= fZapCr0;
5460	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5461
5462	/* Commit the CR0 and related fields to the guest VMCS. */
5463	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5464	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5465	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5466	{
5467	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5468	AssertRC(rc);
5469	}
5470	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5471	{
5472	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5473	AssertRC(rc);
5474	}
5475
5476	/* Update our caches. */
5477	pVmcsInfo->u32ProcCtls = uProcCtls;
5478	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5479
5480	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5481	}
5482	else
5483	{
5484	/*
5485	* With nested-guests, we may have extended the guest/host mask here since we
5486	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5487	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
5488	* originally supplied. We must copy those bits from the nested-guest CR0 into
5489	* the nested-guest CR0 read-shadow.
5490	*/
5491	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5492	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5493	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5494	Assert(!RT_HI_U32(u64GuestCr0));
5495	Assert(u64GuestCr0 & X86_CR0_NE);
5496
5497	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5498	u64GuestCr0 \|= fSetCr0;
5499	u64GuestCr0 &= fZapCr0;
5500	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5501
5502	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5503	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5504	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5505
5506	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5507	}
5508
5509	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5510	}
5511
5512	return VINF_SUCCESS;
5513	}
5514
5515
5516	/**
5517	* Exports the guest control registers (CR3, CR4) into the guest-state area
5518	* in the VMCS.
5519	*
5520	* @returns VBox strict status code.
5521	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5522	* without unrestricted guest access and the VMMDev is not presently
5523	* mapped (e.g. EFI32).
5524	*
5525	* @param pVCpu The cross context virtual CPU structure.
5526	* @param pVmxTransient The VMX-transient structure.
5527	*
5528	* @remarks No-long-jump zone!!!
5529	*/
5530	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5531	{
5532	int rc = VINF_SUCCESS;
5533	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5534
5535	/*
5536	* Guest CR2.
5537	* It's always loaded in the assembler code. Nothing to do here.
5538	*/
5539
5540	/*
5541	* Guest CR3.
5542	*/
5543	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5544	{
5545	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5546
5547	if (pVM->hm.s.fNestedPaging)
5548	{
5549	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5550	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5551
5552	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5553	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5554	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5555	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5556
5557	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5558	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5559	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5560
5561	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5562	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5563	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5564	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5565	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5566	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5567	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5568
5569	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5570	AssertRC(rc);
5571
5572	uint64_t u64GuestCr3;
5573	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5574	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5575	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5576	{
5577	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5578	if (CPUMIsGuestInPAEModeEx(pCtx))
5579	{
5580	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5581	AssertRC(rc);
5582	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
5583	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
5584	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
5585	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
5586	}
5587
5588	/*
5589	* The guest's view of its CR3 is unblemished with nested paging when the
5590	* guest is using paging or we have unrestricted guest execution to handle
5591	* the guest when it's not using paging.
5592	*/
5593	u64GuestCr3 = pCtx->cr3;
5594	}
5595	else
5596	{
5597	/*
5598	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5599	* thinks it accesses physical memory directly, we use our identity-mapped
5600	* page table to map guest-linear to guest-physical addresses. EPT takes care
5601	* of translating it to host-physical addresses.
5602	*/
5603	RTGCPHYS GCPhys;
5604	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5605
5606	/* We obtain it here every time as the guest could have relocated this PCI region. */
5607	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5608	if (RT_SUCCESS(rc))
5609	{ /* likely */ }
5610	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5611	{
5612	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5613	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5614	}
5615	else
5616	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5617
5618	u64GuestCr3 = GCPhys;
5619	}
5620
5621	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
5622	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, u64GuestCr3);
5623	AssertRC(rc);
5624	}
5625	else
5626	{
5627	Assert(!pVmxTransient->fIsNestedGuest);
5628	/* Non-nested paging case, just use the hypervisor's CR3. */
5629	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5630
5631	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
5632	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5633	AssertRC(rc);
5634	}
5635
5636	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5637	}
5638
5639	/*
5640	* Guest CR4.
5641	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5642	*/
5643	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5644	{
5645	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5646	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5647
5648	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0;
5649	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5650
5651	/*
5652	* With nested-guests, we may have extended the guest/host mask here (since we
5653	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5654	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5655	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
5656	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5657	*/
5658	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5659	uint64_t u64GuestCr4 = pCtx->cr4;
5660	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5661	? pCtx->cr4
5662	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5663	Assert(!RT_HI_U32(u64GuestCr4));
5664
5665	/*
5666	* Setup VT-x's view of the guest CR4.
5667	*
5668	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5669	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5670	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5671	*
5672	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5673	*/
5674	if (pVmcsInfo->RealMode.fRealOnV86Active)
5675	{
5676	Assert(pVM->hm.s.vmx.pRealModeTSS);
5677	Assert(PDMVmmDevHeapIsEnabled(pVM));
5678	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5679	}
5680
5681	if (pVM->hm.s.fNestedPaging)
5682	{
5683	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5684	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5685	{
5686	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5687	u64GuestCr4 \|= X86_CR4_PSE;
5688	/* Our identity mapping is a 32-bit page directory. */
5689	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5690	}
5691	/* else use guest CR4.*/
5692	}
5693	else
5694	{
5695	Assert(!pVmxTransient->fIsNestedGuest);
5696
5697	/*
5698	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5699	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5700	*/
5701	switch (pVCpu->hm.s.enmShadowMode)
5702	{
5703	case PGMMODE_REAL: /* Real-mode. */
5704	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5705	case PGMMODE_32_BIT: /* 32-bit paging. */
5706	{
5707	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5708	break;
5709	}
5710
5711	case PGMMODE_PAE: /* PAE paging. */
5712	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5713	{
5714	u64GuestCr4 \|= X86_CR4_PAE;
5715	break;
5716	}
5717
5718	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5719	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5720	{
5721	#ifdef VBOX_WITH_64_BITS_GUESTS
5722	/* For our assumption in hmR0VmxShouldSwapEferMsr. */
5723	Assert(u64GuestCr4 & X86_CR4_PAE);
5724	break;
5725	#endif
5726	}
5727	default:
5728	AssertFailed();
5729	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5730	}
5731	}
5732
5733	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
5734	u64GuestCr4 \|= fSetCr4;
5735	u64GuestCr4 &= fZapCr4;
5736
5737	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5738	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
5739	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
5740
5741	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5742	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5743
5744	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5745
5746	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5747	}
5748	return rc;
5749	}
5750
5751
5752	/**
5753	* Exports the guest debug registers into the guest-state area in the VMCS.
5754	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5755	*
5756	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5757	*
5758	* @returns VBox status code.
5759	* @param pVCpu The cross context virtual CPU structure.
5760	* @param pVmxTransient The VMX-transient structure.
5761	*
5762	* @remarks No-long-jump zone!!!
5763	*/
5764	static int hmR0VmxExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5765	{
5766	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5767
5768	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5769	* stepping. */
5770	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5771	if (pVmxTransient->fIsNestedGuest)
5772	{
5773	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5774	AssertRC(rc);
5775
5776	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5777	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5778	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5779	AssertRC(rc);
5780	return VINF_SUCCESS;
5781	}
5782
5783	#ifdef VBOX_STRICT
5784	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5785	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5786	{
5787	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5788	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5789	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5790	}
5791	#endif
5792
5793	bool fSteppingDB = false;
5794	bool fInterceptMovDRx = false;
5795	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5796	if (pVCpu->hm.s.fSingleInstruction)
5797	{
5798	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5799	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5800	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5801	{
5802	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5803	Assert(fSteppingDB == false);
5804	}
5805	else
5806	{
5807	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5808	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5809	pVCpu->hm.s.fClearTrapFlag = true;
5810	fSteppingDB = true;
5811	}
5812	}
5813
5814	uint64_t u64GuestDr7;
5815	if ( fSteppingDB
5816	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5817	{
5818	/*
5819	* Use the combined guest and host DRx values found in the hypervisor register set
5820	* because the hypervisor debugger has breakpoints active or someone is single stepping
5821	* on the host side without a monitor trap flag.
5822	*
5823	* Note! DBGF expects a clean DR6 state before executing guest code.
5824	*/
5825	if (!CPUMIsHyperDebugStateActive(pVCpu))
5826	{
5827	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5828	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5829	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5830	}
5831
5832	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5833	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
5834	pVCpu->hm.s.fUsingHyperDR7 = true;
5835	fInterceptMovDRx = true;
5836	}
5837	else
5838	{
5839	/*
5840	* If the guest has enabled debug registers, we need to load them prior to
5841	* executing guest code so they'll trigger at the right time.
5842	*/
5843	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5844	{
5845	if (!CPUMIsGuestDebugStateActive(pVCpu))
5846	{
5847	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5848	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5849	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5850	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5851	}
5852	Assert(!fInterceptMovDRx);
5853	}
5854	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5855	{
5856	/*
5857	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5858	* must intercept #DB in order to maintain a correct DR6 guest value, and
5859	* because we need to intercept it to prevent nested #DBs from hanging the
5860	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5861	*/
5862	fInterceptMovDRx = true;
5863	}
5864
5865	/* Update DR7 with the actual guest value. */
5866	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5867	pVCpu->hm.s.fUsingHyperDR7 = false;
5868	}
5869
5870	if (fInterceptMovDRx)
5871	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5872	else
5873	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5874
5875	/*
5876	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5877	* monitor-trap flag and update our cache.
5878	*/
5879	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5880	{
5881	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5882	AssertRC(rc);
5883	pVmcsInfo->u32ProcCtls = uProcCtls;
5884	}
5885
5886	/*
5887	* Update guest DR7.
5888	*/
5889	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
5890	AssertRC(rc);
5891
5892	/*
5893	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5894	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5895	*
5896	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5897	*/
5898	if (fSteppingDB)
5899	{
5900	Assert(pVCpu->hm.s.fSingleInstruction);
5901	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5902
5903	uint32_t fIntrState = 0;
5904	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5905	AssertRC(rc);
5906
5907	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5908	{
5909	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5910	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5911	AssertRC(rc);
5912	}
5913	}
5914
5915	return VINF_SUCCESS;
5916	}
5917
5918
5919	#ifdef VBOX_STRICT
5920	/**
5921	* Strict function to validate segment registers.
5922	*
5923	* @param pVCpu The cross context virtual CPU structure.
5924	* @param pVmcsInfo The VMCS info. object.
5925	*
5926	* @remarks Will import guest CR0 on strict builds during validation of
5927	* segments.
5928	*/
5929	static void hmR0VmxValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5930	{
5931	/*
5932	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5933	*
5934	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5935	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5936	* unusable bit and doesn't change the guest-context value.
5937	*/
5938	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5939	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5940	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5941	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5942	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5943	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5944	{
5945	/* Protected mode checks */
5946	/* CS */
5947	Assert(pCtx->cs.Attr.n.u1Present);
5948	Assert(!(pCtx->cs.Attr.u & 0xf00));
5949	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5950	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5951	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5952	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5953	\|\| (pCtx->cs.Attr.n.u1Granularity));
5954	/* CS cannot be loaded with NULL in protected mode. */
5955	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5956	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5957	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5958	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5959	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5960	else
5961	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5962	/* SS */
5963	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5964	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5965	if ( !(pCtx->cr0 & X86_CR0_PE)
5966	\|\| pCtx->cs.Attr.n.u4Type == 3)
5967	{
5968	Assert(!pCtx->ss.Attr.n.u2Dpl);
5969	}
5970	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5971	{
5972	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5973	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5974	Assert(pCtx->ss.Attr.n.u1Present);
5975	Assert(!(pCtx->ss.Attr.u & 0xf00));
5976	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5977	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5978	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5979	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5980	\|\| (pCtx->ss.Attr.n.u1Granularity));
5981	}
5982	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5983	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5984	{
5985	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5986	Assert(pCtx->ds.Attr.n.u1Present);
5987	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5988	Assert(!(pCtx->ds.Attr.u & 0xf00));
5989	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5990	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5991	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5992	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5993	\|\| (pCtx->ds.Attr.n.u1Granularity));
5994	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5995	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5996	}
5997	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5998	{
5999	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6000	Assert(pCtx->es.Attr.n.u1Present);
6001	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6002	Assert(!(pCtx->es.Attr.u & 0xf00));
6003	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6004	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6005	\|\| !(pCtx->es.Attr.n.u1Granularity));
6006	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6007	\|\| (pCtx->es.Attr.n.u1Granularity));
6008	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6009	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6010	}
6011	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6012	{
6013	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6014	Assert(pCtx->fs.Attr.n.u1Present);
6015	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6016	Assert(!(pCtx->fs.Attr.u & 0xf00));
6017	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6018	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6019	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6020	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6021	\|\| (pCtx->fs.Attr.n.u1Granularity));
6022	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6023	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6024	}
6025	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6026	{
6027	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6028	Assert(pCtx->gs.Attr.n.u1Present);
6029	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6030	Assert(!(pCtx->gs.Attr.u & 0xf00));
6031	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6032	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6033	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6034	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6035	\|\| (pCtx->gs.Attr.n.u1Granularity));
6036	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6037	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6038	}
6039	/* 64-bit capable CPUs. */
6040	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6041	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6042	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6043	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6044	}
6045	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6046	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6047	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6048	{
6049	/* Real and v86 mode checks. */
6050	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6051	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6052	if (pVmcsInfo->RealMode.fRealOnV86Active)
6053	{
6054	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6055	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6056	}
6057	else
6058	{
6059	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6060	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6061	}
6062
6063	/* CS */
6064	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6065	Assert(pCtx->cs.u32Limit == 0xffff);
6066	Assert(u32CSAttr == 0xf3);
6067	/* SS */
6068	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6069	Assert(pCtx->ss.u32Limit == 0xffff);
6070	Assert(u32SSAttr == 0xf3);
6071	/* DS */
6072	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6073	Assert(pCtx->ds.u32Limit == 0xffff);
6074	Assert(u32DSAttr == 0xf3);
6075	/* ES */
6076	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6077	Assert(pCtx->es.u32Limit == 0xffff);
6078	Assert(u32ESAttr == 0xf3);
6079	/* FS */
6080	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6081	Assert(pCtx->fs.u32Limit == 0xffff);
6082	Assert(u32FSAttr == 0xf3);
6083	/* GS */
6084	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6085	Assert(pCtx->gs.u32Limit == 0xffff);
6086	Assert(u32GSAttr == 0xf3);
6087	/* 64-bit capable CPUs. */
6088	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6089	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6090	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6091	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6092	}
6093	}
6094	#endif /* VBOX_STRICT */
6095
6096
6097	/**
6098	* Exports a guest segment register into the guest-state area in the VMCS.
6099	*
6100	* @returns VBox status code.
6101	* @param pVCpu The cross context virtual CPU structure.
6102	* @param pVmcsInfo The VMCS info. object.
6103	* @param iSegReg The segment register number (X86_SREG_XXX).
6104	* @param pSelReg Pointer to the segment selector.
6105	*
6106	* @remarks No-long-jump zone!!!
6107	*/
6108	static int hmR0VmxExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6109	{
6110	Assert(iSegReg < X86_SREG_COUNT);
6111	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6112	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6113	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6114	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6115
6116	uint32_t u32Access = pSelReg->Attr.u;
6117	if (pVmcsInfo->RealMode.fRealOnV86Active)
6118	{
6119	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6120	u32Access = 0xf3;
6121	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6122	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6123	RT_NOREF_PV(pVCpu);
6124	}
6125	else
6126	{
6127	/*
6128	* The way to differentiate between whether this is really a null selector or was just
6129	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6130	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6131	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6132	* NULL selectors loaded in protected-mode have their attribute as 0.
6133	*/
6134	if (!u32Access)
6135	u32Access = X86DESCATTR_UNUSABLE;
6136	}
6137
6138	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6139	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6140	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6141
6142	/*
6143	* Commit it to the VMCS.
6144	*/
6145	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); AssertRC(rc);
6146	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); AssertRC(rc);
6147	rc = VMXWriteVmcsNw(idxBase, pSelReg->u64Base); AssertRC(rc);
6148	rc = VMXWriteVmcs32(idxAttr, u32Access); AssertRC(rc);
6149	return VINF_SUCCESS;
6150	}
6151
6152
6153	/**
6154	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6155	* area in the VMCS.
6156	*
6157	* @returns VBox status code.
6158	* @param pVCpu The cross context virtual CPU structure.
6159	* @param pVmxTransient The VMX-transient structure.
6160	*
6161	* @remarks Will import guest CR0 on strict builds during validation of
6162	* segments.
6163	* @remarks No-long-jump zone!!!
6164	*/
6165	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6166	{
6167	int rc = VERR_INTERNAL_ERROR_5;
6168	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6169	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6170	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6171
6172	/*
6173	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6174	*/
6175	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6176	{
6177	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6178	{
6179	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6180	if (pVmcsInfo->RealMode.fRealOnV86Active)
6181	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6182	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6183	AssertRC(rc);
6184	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6185	}
6186
6187	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6188	{
6189	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6190	if (pVmcsInfo->RealMode.fRealOnV86Active)
6191	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6192	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6193	AssertRC(rc);
6194	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6195	}
6196
6197	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6198	{
6199	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6200	if (pVmcsInfo->RealMode.fRealOnV86Active)
6201	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6202	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6203	AssertRC(rc);
6204	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6205	}
6206
6207	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6208	{
6209	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6210	if (pVmcsInfo->RealMode.fRealOnV86Active)
6211	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6212	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6213	AssertRC(rc);
6214	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6215	}
6216
6217	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6218	{
6219	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6220	if (pVmcsInfo->RealMode.fRealOnV86Active)
6221	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6222	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6223	AssertRC(rc);
6224	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6225	}
6226
6227	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6228	{
6229	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6230	if (pVmcsInfo->RealMode.fRealOnV86Active)
6231	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6232	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6233	AssertRC(rc);
6234	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6235	}
6236
6237	#ifdef VBOX_STRICT
6238	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6239	#endif
6240	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6241	pCtx->cs.Attr.u));
6242	}
6243
6244	/*
6245	* Guest TR.
6246	*/
6247	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6248	{
6249	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6250
6251	/*
6252	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6253	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6254	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6255	*/
6256	uint16_t u16Sel;
6257	uint32_t u32Limit;
6258	uint64_t u64Base;
6259	uint32_t u32AccessRights;
6260	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6261	{
6262	u16Sel = pCtx->tr.Sel;
6263	u32Limit = pCtx->tr.u32Limit;
6264	u64Base = pCtx->tr.u64Base;
6265	u32AccessRights = pCtx->tr.Attr.u;
6266	}
6267	else
6268	{
6269	Assert(!pVmxTransient->fIsNestedGuest);
6270	Assert(pVM->hm.s.vmx.pRealModeTSS);
6271	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6272
6273	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6274	RTGCPHYS GCPhys;
6275	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6276	AssertRCReturn(rc, rc);
6277
6278	X86DESCATTR DescAttr;
6279	DescAttr.u = 0;
6280	DescAttr.n.u1Present = 1;
6281	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6282
6283	u16Sel = 0;
6284	u32Limit = HM_VTX_TSS_SIZE;
6285	u64Base = GCPhys;
6286	u32AccessRights = DescAttr.u;
6287	}
6288
6289	/* Validate. */
6290	Assert(!(u16Sel & RT_BIT(2)));
6291	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6292	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6293	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6294	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6295	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6296	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6297	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6298	Assert( (u32Limit & 0xfff) == 0xfff
6299	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6300	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6301	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6302
6303	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
6304	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
6305	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
6306	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
6307
6308	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6309	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6310	}
6311
6312	/*
6313	* Guest GDTR.
6314	*/
6315	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6316	{
6317	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6318
6319	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
6320	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
6321
6322	/* Validate. */
6323	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6324
6325	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6326	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6327	}
6328
6329	/*
6330	* Guest LDTR.
6331	*/
6332	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6333	{
6334	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6335
6336	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6337	uint32_t u32Access;
6338	if ( !pVmxTransient->fIsNestedGuest
6339	&& !pCtx->ldtr.Attr.u)
6340	u32Access = X86DESCATTR_UNUSABLE;
6341	else
6342	u32Access = pCtx->ldtr.Attr.u;
6343
6344	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
6345	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
6346	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
6347	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
6348
6349	/* Validate. */
6350	if (!(u32Access & X86DESCATTR_UNUSABLE))
6351	{
6352	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6353	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6354	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6355	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6356	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6357	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6358	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6359	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6360	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6361	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6362	}
6363
6364	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6365	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6366	}
6367
6368	/*
6369	* Guest IDTR.
6370	*/
6371	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6372	{
6373	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6374
6375	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
6376	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
6377
6378	/* Validate. */
6379	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6380
6381	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6382	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6383	}
6384
6385	return VINF_SUCCESS;
6386	}
6387
6388
6389	/**
6390	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6391	* areas.
6392	*
6393	* These MSRs will automatically be loaded to the host CPU on every successful
6394	* VM-entry and stored from the host CPU on every successful VM-exit.
6395	*
6396	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6397	* actual host MSR values are not- updated here for performance reasons. See
6398	* hmR0VmxExportHostMsrs().
6399	*
6400	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6401	*
6402	* @returns VBox status code.
6403	* @param pVCpu The cross context virtual CPU structure.
6404	* @param pVmxTransient The VMX-transient structure.
6405	*
6406	* @remarks No-long-jump zone!!!
6407	*/
6408	static int hmR0VmxExportGuestMsrs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6409	{
6410	AssertPtr(pVCpu);
6411	AssertPtr(pVmxTransient);
6412
6413	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6414	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6415
6416	/*
6417	* MSRs that we use the auto-load/store MSR area in the VMCS.
6418	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
6419	* nothing to do here. The host MSR values are updated when it's safe in
6420	* hmR0VmxLazySaveHostMsrs().
6421	*
6422	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6423	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6424	* emulation. The merged MSR permission bitmap will ensure that we get VM-exits
6425	* for any MSR that are not part of the lazy MSRs so we do not need to place
6426	* those MSRs into the auto-load/store MSR area. Nothing to do here.
6427	*/
6428	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6429	{
6430	/* No auto-load/store MSRs currently. */
6431	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6432	}
6433
6434	/*
6435	* Guest Sysenter MSRs.
6436	*/
6437	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6438	{
6439	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6440
6441	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6442	{
6443	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6444	AssertRC(rc);
6445	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6446	}
6447
6448	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6449	{
6450	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6451	AssertRC(rc);
6452	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6453	}
6454
6455	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6456	{
6457	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6458	AssertRC(rc);
6459	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6460	}
6461	}
6462
6463	/*
6464	* Guest/host EFER MSR.
6465	*/
6466	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6467	{
6468	/* Whether we are using the VMCS to swap the EFER MSR must have been
6469	determined earlier while exporting VM-entry/VM-exit controls. */
6470	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6471	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6472
6473	if (hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
6474	{
6475	/*
6476	* EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
6477	* This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
6478	* VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
6479	* and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
6480	* must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
6481	* the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
6482	* during VM-entry.
6483	*/
6484	uint64_t uGuestEferMsr = pCtx->msrEFER;
6485	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6486	{
6487	if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
6488	uGuestEferMsr &= ~MSR_K6_EFER_LME;
6489	else
6490	Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
6491	}
6492
6493	/*
6494	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6495	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6496	*/
6497	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6498	{
6499	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
6500	AssertRC(rc);
6501	}
6502	else
6503	{
6504	/*
6505	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6506	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6507	*/
6508	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
6509	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6510	AssertRCReturn(rc, rc);
6511	}
6512
6513	Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
6514	}
6515	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6516	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6517
6518	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6519	}
6520
6521	/*
6522	* Other MSRs.
6523	* Speculation Control (R/W).
6524	*/
6525	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6526	{
6527	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6528	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6529	{
6530	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6531	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6532	AssertRCReturn(rc, rc);
6533	}
6534	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6535	}
6536
6537	return VINF_SUCCESS;
6538	}
6539
6540
6541	/**
6542	* Wrapper for running the guest code in VT-x.
6543	*
6544	* @returns VBox status code, no informational status codes.
6545	* @param pVCpu The cross context virtual CPU structure.
6546	* @param pVmxTransient The VMX-transient structure.
6547	*
6548	* @remarks No-long-jump zone!!!
6549	*/
6550	DECLINLINE(int) hmR0VmxRunGuest(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6551	{
6552	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6553	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6554	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6555
6556	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6557
6558	/*
6559	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6560	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6561	* callee-saved and thus the need for this XMM wrapper.
6562	*
6563	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6564	*/
6565	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6566	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6567	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6568	#ifdef VBOX_WITH_KERNEL_USING_XMM
6569	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6570	#else
6571	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu);
6572	#endif
6573	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6574	return rc;
6575	}
6576
6577
6578	/**
6579	* Reports world-switch error and dumps some useful debug info.
6580	*
6581	* @param pVCpu The cross context virtual CPU structure.
6582	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6583	* @param pVmxTransient The VMX-transient structure (only
6584	* exitReason updated).
6585	*/
6586	static void hmR0VmxReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6587	{
6588	Assert(pVCpu);
6589	Assert(pVmxTransient);
6590	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6591
6592	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6593	switch (rcVMRun)
6594	{
6595	case VERR_VMX_INVALID_VMXON_PTR:
6596	AssertFailed();
6597	break;
6598	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6599	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6600	{
6601	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6602	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6603	AssertRC(rc);
6604	hmR0VmxReadExitQualVmcs(pVmxTransient);
6605
6606	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6607	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6608	Cannot do it here as we may have been long preempted. */
6609
6610	#ifdef VBOX_STRICT
6611	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6612	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6613	pVmxTransient->uExitReason));
6614	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6615	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6616	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6617	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6618	else
6619	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6620	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6621	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6622
6623	static struct
6624	{
6625	/** Name of the field to log. */
6626	const char *pszName;
6627	/** The VMCS field. */
6628	uint32_t uVmcsField;
6629	/** Whether host support of this field needs to be checked. */
6630	bool fCheckSupport;
6631	} const s_aVmcsFields[] =
6632	{
6633	{ "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false },
6634	{ "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false },
6635	{ "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true },
6636	{ "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false },
6637	{ "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false },
6638	{ "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false },
6639	{ "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false },
6640	{ "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false },
6641	{ "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false },
6642	{ "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false },
6643	{ "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false },
6644	{ "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false },
6645	{ "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false },
6646	{ "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false },
6647	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false },
6648	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false },
6649	{ "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false },
6650	{ "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false },
6651	{ "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false },
6652	{ "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false },
6653	{ "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true },
6654	{ "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false },
6655	{ "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false },
6656	{ "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false },
6657	{ "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
6658	{ "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false },
6659	{ "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false },
6660	{ "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false },
6661	/* The order of selector fields below are fixed! */
6662	{ "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false },
6663	{ "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false },
6664	{ "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false },
6665	{ "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false },
6666	{ "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false },
6667	{ "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false },
6668	{ "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false },
6669	/* End of ordered selector fields. */
6670	{ "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false },
6671	{ "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false },
6672	{ "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false },
6673	{ "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false },
6674	{ "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false },
6675	{ "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false },
6676	{ "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false },
6677	{ "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false }
6678	};
6679
6680	RTGDTR HostGdtr;
6681	ASMGetGDTR(&HostGdtr);
6682
6683	uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
6684	for (uint32_t i = 0; i < cVmcsFields; i++)
6685	{
6686	uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
6687
6688	bool fSupported;
6689	if (!s_aVmcsFields[i].fCheckSupport)
6690	fSupported = true;
6691	else
6692	{
6693	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6694	switch (uVmcsField)
6695	{
6696	case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hm.s.fNestedPaging; break;
6697	case VMX_VMCS16_VPID: fSupported = pVM->hm.s.vmx.fVpid; break;
6698	case VMX_VMCS32_CTRL_PROC_EXEC2:
6699	fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6700	break;
6701	default:
6702	AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
6703	}
6704	}
6705
6706	if (fSupported)
6707	{
6708	uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
6709	switch (uWidth)
6710	{
6711	case VMX_VMCSFIELD_WIDTH_16BIT:
6712	{
6713	uint16_t u16Val;
6714	rc = VMXReadVmcs16(uVmcsField, &u16Val);
6715	AssertRC(rc);
6716	Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
6717
6718	if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
6719	&& uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
6720	{
6721	if (u16Val < HostGdtr.cbGdt)
6722	{
6723	/* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
6724	static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
6725	"Host FS", "Host GS", "Host TR" };
6726	uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
6727	Assert(idxSel < RT_ELEMENTS(s_apszSel));
6728	PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
6729	hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
6730	}
6731	else
6732	Log4((" Selector value exceeds GDT limit!\n"));
6733	}
6734	break;
6735	}
6736
6737	case VMX_VMCSFIELD_WIDTH_32BIT:
6738	{
6739	uint32_t u32Val;
6740	rc = VMXReadVmcs32(uVmcsField, &u32Val);
6741	AssertRC(rc);
6742	Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
6743	break;
6744	}
6745
6746	case VMX_VMCSFIELD_WIDTH_64BIT:
6747	case VMX_VMCSFIELD_WIDTH_NATURAL:
6748	{
6749	uint64_t u64Val;
6750	rc = VMXReadVmcs64(uVmcsField, &u64Val);
6751	AssertRC(rc);
6752	Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
6753	break;
6754	}
6755	}
6756	}
6757	}
6758
6759	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6760	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6761	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6762	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6763	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6764	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6765	#endif /* VBOX_STRICT */
6766	break;
6767	}
6768
6769	default:
6770	/* Impossible */
6771	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6772	break;
6773	}
6774	}
6775
6776
6777	/**
6778	* Sets up the usage of TSC-offsetting and updates the VMCS.
6779	*
6780	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6781	* VMX-preemption timer.
6782	*
6783	* @returns VBox status code.
6784	* @param pVCpu The cross context virtual CPU structure.
6785	* @param pVmxTransient The VMX-transient structure.
6786	*
6787	* @remarks No-long-jump zone!!!
6788	*/
6789	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6790	{
6791	bool fOffsettedTsc;
6792	bool fParavirtTsc;
6793	uint64_t uTscOffset;
6794	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6795	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6796
6797	if (pVM->hm.s.vmx.fUsePreemptTimer)
6798	{
6799	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6800
6801	/* Make sure the returned values have sane upper and lower boundaries. */
6802	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6803	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6804	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6805	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6806
6807	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6808	* preemption timers here. We probably need to clamp the preemption timer,
6809	* after converting the timer value to the host. */
6810	uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6811	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6812	AssertRC(rc);
6813	}
6814	else
6815	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6816
6817	if (fParavirtTsc)
6818	{
6819	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6820	information before every VM-entry, hence disable it for performance sake. */
6821	#if 0
6822	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6823	AssertRC(rc);
6824	#endif
6825	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6826	}
6827
6828	if ( fOffsettedTsc
6829	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6830	{
6831	if (pVmxTransient->fIsNestedGuest)
6832	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6833	hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
6834	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6835	}
6836	else
6837	{
6838	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6839	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6840	}
6841	}
6842
6843
6844	/**
6845	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6846	* VM-exit interruption info type.
6847	*
6848	* @returns The IEM exception flags.
6849	* @param uVector The event vector.
6850	* @param uVmxEventType The VMX event type.
6851	*
6852	* @remarks This function currently only constructs flags required for
6853	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6854	* and CR2 aspects of an exception are not included).
6855	*/
6856	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6857	{
6858	uint32_t fIemXcptFlags;
6859	switch (uVmxEventType)
6860	{
6861	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6862	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6863	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6864	break;
6865
6866	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6867	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6868	break;
6869
6870	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6871	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6872	break;
6873
6874	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6875	{
6876	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6877	if (uVector == X86_XCPT_BP)
6878	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6879	else if (uVector == X86_XCPT_OF)
6880	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6881	else
6882	{
6883	fIemXcptFlags = 0;
6884	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6885	}
6886	break;
6887	}
6888
6889	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6890	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6891	break;
6892
6893	default:
6894	fIemXcptFlags = 0;
6895	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6896	break;
6897	}
6898	return fIemXcptFlags;
6899	}
6900
6901
6902	/**
6903	* Sets an event as a pending event to be injected into the guest.
6904	*
6905	* @param pVCpu The cross context virtual CPU structure.
6906	* @param u32IntInfo The VM-entry interruption-information field.
6907	* @param cbInstr The VM-entry instruction length in bytes (for
6908	* software interrupts, exceptions and privileged
6909	* software exceptions).
6910	* @param u32ErrCode The VM-entry exception error code.
6911	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6912	* page-fault.
6913	*/
6914	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6915	RTGCUINTPTR GCPtrFaultAddress)
6916	{
6917	Assert(!pVCpu->hm.s.Event.fPending);
6918	pVCpu->hm.s.Event.fPending = true;
6919	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6920	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6921	pVCpu->hm.s.Event.cbInstr = cbInstr;
6922	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6923	}
6924
6925
6926	/**
6927	* Sets an external interrupt as pending-for-injection into the VM.
6928	*
6929	* @param pVCpu The cross context virtual CPU structure.
6930	* @param u8Interrupt The external interrupt vector.
6931	*/
6932	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
6933	{
6934	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6935	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6936	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6937	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6938	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6939	}
6940
6941
6942	/**
6943	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6944	*
6945	* @param pVCpu The cross context virtual CPU structure.
6946	*/
6947	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPUCC pVCpu)
6948	{
6949	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6950	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6951	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6952	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6953	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6954	}
6955
6956
6957	/**
6958	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6959	*
6960	* @param pVCpu The cross context virtual CPU structure.
6961	*/
6962	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPUCC pVCpu)
6963	{
6964	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6965	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6966	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6967	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6968	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6969	}
6970
6971
6972	/**
6973	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6974	*
6975	* @param pVCpu The cross context virtual CPU structure.
6976	*/
6977	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPUCC pVCpu)
6978	{
6979	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6980	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6981	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6982	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6983	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6984	}
6985
6986
6987	/**
6988	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6989	*
6990	* @param pVCpu The cross context virtual CPU structure.
6991	*/
6992	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPUCC pVCpu)
6993	{
6994	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6995	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6996	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6997	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6998	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6999	}
7000
7001
7002	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7003	/**
7004	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7005	*
7006	* @param pVCpu The cross context virtual CPU structure.
7007	* @param u32ErrCode The error code for the general-protection exception.
7008	*/
7009	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7010	{
7011	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7012	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7013	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7014	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7015	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7016	}
7017
7018
7019	/**
7020	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7021	*
7022	* @param pVCpu The cross context virtual CPU structure.
7023	* @param u32ErrCode The error code for the stack exception.
7024	*/
7025	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7026	{
7027	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7028	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7029	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7030	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7031	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7032	}
7033	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7034
7035
7036	/**
7037	* Fixes up attributes for the specified segment register.
7038	*
7039	* @param pVCpu The cross context virtual CPU structure.
7040	* @param pSelReg The segment register that needs fixing.
7041	* @param idxSel The VMCS field for the corresponding segment register.
7042	*/
7043	static void hmR0VmxFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7044	{
7045	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7046
7047	/*
7048	* If VT-x marks the segment as unusable, most other bits remain undefined:
7049	* - For CS the L, D and G bits have meaning.
7050	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7051	* - For the remaining data segments no bits are defined.
7052	*
7053	* The present bit and the unusable bit has been observed to be set at the
7054	* same time (the selector was supposed to be invalid as we started executing
7055	* a V8086 interrupt in ring-0).
7056	*
7057	* What should be important for the rest of the VBox code, is that the P bit is
7058	* cleared. Some of the other VBox code recognizes the unusable bit, but
7059	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7060	* safe side here, we'll strip off P and other bits we don't care about. If
7061	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7062	*
7063	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7064	*/
7065	#ifdef VBOX_STRICT
7066	uint32_t const uAttr = pSelReg->Attr.u;
7067	#endif
7068
7069	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7070	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7071	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7072
7073	#ifdef VBOX_STRICT
7074	VMMRZCallRing3Disable(pVCpu);
7075	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7076	# ifdef DEBUG_bird
7077	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7078	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7079	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7080	# endif
7081	VMMRZCallRing3Enable(pVCpu);
7082	NOREF(uAttr);
7083	#endif
7084	RT_NOREF2(pVCpu, idxSel);
7085	}
7086
7087
7088	/**
7089	* Imports a guest segment register from the current VMCS into the guest-CPU
7090	* context.
7091	*
7092	* @param pVCpu The cross context virtual CPU structure.
7093	* @param iSegReg The segment register number (X86_SREG_XXX).
7094	*
7095	* @remarks Called with interrupts and/or preemption disabled.
7096	*/
7097	static void hmR0VmxImportGuestSegReg(PVMCPUCC pVCpu, uint8_t iSegReg)
7098	{
7099	Assert(iSegReg < X86_SREG_COUNT);
7100
7101	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7102	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7103	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7104	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7105
7106	uint16_t u16Sel;
7107	uint64_t u64Base;
7108	uint32_t u32Limit, u32Attr;
7109	int rc = VMXReadVmcs16(idxSel, &u16Sel); AssertRC(rc);
7110	rc = VMXReadVmcs32(idxLimit, &u32Limit); AssertRC(rc);
7111	rc = VMXReadVmcs32(idxAttr, &u32Attr); AssertRC(rc);
7112	rc = VMXReadVmcsNw(idxBase, &u64Base); AssertRC(rc);
7113
7114	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7115	pSelReg->Sel = u16Sel;
7116	pSelReg->ValidSel = u16Sel;
7117	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7118	pSelReg->u32Limit = u32Limit;
7119	pSelReg->u64Base = u64Base;
7120	pSelReg->Attr.u = u32Attr;
7121	if (u32Attr & X86DESCATTR_UNUSABLE)
7122	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7123	}
7124
7125
7126	/**
7127	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7128	*
7129	* @param pVCpu The cross context virtual CPU structure.
7130	*
7131	* @remarks Called with interrupts and/or preemption disabled.
7132	*/
7133	static void hmR0VmxImportGuestLdtr(PVMCPUCC pVCpu)
7134	{
7135	uint16_t u16Sel;
7136	uint64_t u64Base;
7137	uint32_t u32Limit, u32Attr;
7138	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
7139	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
7140	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7141	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
7142
7143	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
7144	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
7145	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7146	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7147	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7148	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7149	if (u32Attr & X86DESCATTR_UNUSABLE)
7150	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7151	}
7152
7153
7154	/**
7155	* Imports the guest TR from the current VMCS into the guest-CPU context.
7156	*
7157	* @param pVCpu The cross context virtual CPU structure.
7158	*
7159	* @remarks Called with interrupts and/or preemption disabled.
7160	*/
7161	static void hmR0VmxImportGuestTr(PVMCPUCC pVCpu)
7162	{
7163	uint16_t u16Sel;
7164	uint64_t u64Base;
7165	uint32_t u32Limit, u32Attr;
7166	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
7167	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
7168	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7169	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
7170
7171	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
7172	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
7173	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7174	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7175	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7176	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7177	/* TR is the only selector that can never be unusable. */
7178	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7179	}
7180
7181
7182	/**
7183	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7184	*
7185	* @param pVCpu The cross context virtual CPU structure.
7186	*
7187	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7188	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7189	* instead!!!
7190	*/
7191	static void hmR0VmxImportGuestRip(PVMCPUCC pVCpu)
7192	{
7193	uint64_t u64Val;
7194	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7195	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7196	{
7197	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
7198	AssertRC(rc);
7199
7200	pCtx->rip = u64Val;
7201	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7202	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7203	}
7204	}
7205
7206
7207	/**
7208	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7209	*
7210	* @param pVCpu The cross context virtual CPU structure.
7211	* @param pVmcsInfo The VMCS info. object.
7212	*
7213	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7214	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7215	* instead!!!
7216	*/
7217	static void hmR0VmxImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7218	{
7219	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7220	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7221	{
7222	uint64_t u64Val;
7223	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7224	AssertRC(rc);
7225
7226	pCtx->rflags.u64 = u64Val;
7227	if (pVmcsInfo->RealMode.fRealOnV86Active)
7228	{
7229	pCtx->eflags.Bits.u1VM = 0;
7230	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7231	}
7232	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7233	}
7234	}
7235
7236
7237	/**
7238	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7239	* context.
7240	*
7241	* @param pVCpu The cross context virtual CPU structure.
7242	* @param pVmcsInfo The VMCS info. object.
7243	*
7244	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7245	* do not log!
7246	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7247	* instead!!!
7248	*/
7249	static void hmR0VmxImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7250	{
7251	uint32_t u32Val;
7252	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
7253	if (!u32Val)
7254	{
7255	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7256	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7257	CPUMSetGuestNmiBlocking(pVCpu, false);
7258	}
7259	else
7260	{
7261	/*
7262	* We must import RIP here to set our EM interrupt-inhibited state.
7263	* We also import RFLAGS as our code that evaluates pending interrupts
7264	* before VM-entry requires it.
7265	*/
7266	hmR0VmxImportGuestRip(pVCpu);
7267	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7268
7269	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7270	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7271	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7272	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7273
7274	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7275	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7276	}
7277	}
7278
7279
7280	/**
7281	* Worker for VMXR0ImportStateOnDemand.
7282	*
7283	* @returns VBox status code.
7284	* @param pVCpu The cross context virtual CPU structure.
7285	* @param pVmcsInfo The VMCS info. object.
7286	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7287	*/
7288	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7289	{
7290	int rc = VINF_SUCCESS;
7291	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7292	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7293	uint32_t u32Val;
7294
7295	/*
7296	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7297	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7298	* neither are other host platforms.
7299	*
7300	* Committing this temporarily as it prevents BSOD.
7301	*
7302	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7303	*/
7304	#ifdef RT_OS_WINDOWS
7305	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7306	return VERR_HM_IPE_1;
7307	#endif
7308
7309	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7310
7311	/*
7312	* We disable interrupts to make the updating of the state and in particular
7313	* the fExtrn modification atomic wrt to preemption hooks.
7314	*/
7315	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7316
7317	fWhat &= pCtx->fExtrn;
7318	if (fWhat)
7319	{
7320	do
7321	{
7322	if (fWhat & CPUMCTX_EXTRN_RIP)
7323	hmR0VmxImportGuestRip(pVCpu);
7324
7325	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7326	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7327
7328	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7329	hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7330
7331	if (fWhat & CPUMCTX_EXTRN_RSP)
7332	{
7333	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &pCtx->rsp);
7334	AssertRC(rc);
7335	}
7336
7337	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7338	{
7339	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7340	if (fWhat & CPUMCTX_EXTRN_CS)
7341	{
7342	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7343	hmR0VmxImportGuestRip(pVCpu);
7344	if (fRealOnV86Active)
7345	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7346	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7347	}
7348	if (fWhat & CPUMCTX_EXTRN_SS)
7349	{
7350	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7351	if (fRealOnV86Active)
7352	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7353	}
7354	if (fWhat & CPUMCTX_EXTRN_DS)
7355	{
7356	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7357	if (fRealOnV86Active)
7358	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7359	}
7360	if (fWhat & CPUMCTX_EXTRN_ES)
7361	{
7362	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7363	if (fRealOnV86Active)
7364	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7365	}
7366	if (fWhat & CPUMCTX_EXTRN_FS)
7367	{
7368	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7369	if (fRealOnV86Active)
7370	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7371	}
7372	if (fWhat & CPUMCTX_EXTRN_GS)
7373	{
7374	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7375	if (fRealOnV86Active)
7376	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7377	}
7378	}
7379
7380	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7381	{
7382	if (fWhat & CPUMCTX_EXTRN_LDTR)
7383	hmR0VmxImportGuestLdtr(pVCpu);
7384
7385	if (fWhat & CPUMCTX_EXTRN_GDTR)
7386	{
7387	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
7388	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
7389	pCtx->gdtr.cbGdt = u32Val;
7390	}
7391
7392	/* Guest IDTR. */
7393	if (fWhat & CPUMCTX_EXTRN_IDTR)
7394	{
7395	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
7396	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
7397	pCtx->idtr.cbIdt = u32Val;
7398	}
7399
7400	/* Guest TR. */
7401	if (fWhat & CPUMCTX_EXTRN_TR)
7402	{
7403	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7404	don't need to import that one. */
7405	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7406	hmR0VmxImportGuestTr(pVCpu);
7407	}
7408	}
7409
7410	if (fWhat & CPUMCTX_EXTRN_DR7)
7411	{
7412	if (!pVCpu->hm.s.fUsingHyperDR7)
7413	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &pCtx->dr[7]); AssertRC(rc);
7414	}
7415
7416	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7417	{
7418	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
7419	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
7420	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
7421	pCtx->SysEnter.cs = u32Val;
7422	}
7423
7424	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7425	{
7426	if ( pVM->hm.s.fAllow64BitGuests
7427	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7428	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7429	}
7430
7431	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7432	{
7433	if ( pVM->hm.s.fAllow64BitGuests
7434	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7435	{
7436	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7437	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7438	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7439	}
7440	}
7441
7442	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7443	{
7444	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7445	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7446	Assert(pMsrs);
7447	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7448	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7449	for (uint32_t i = 0; i < cMsrs; i++)
7450	{
7451	uint32_t const idMsr = pMsrs[i].u32Msr;
7452	switch (idMsr)
7453	{
7454	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7455	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7456	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7457	default:
7458	{
7459	pCtx->fExtrn = 0;
7460	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7461	ASMSetFlags(fEFlags);
7462	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7463	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7464	}
7465	}
7466	}
7467	}
7468
7469	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7470	{
7471	if (fWhat & CPUMCTX_EXTRN_CR0)
7472	{
7473	uint64_t u64Cr0;
7474	uint64_t u64Shadow;
7475	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
7476	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
7477	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7478	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7479	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7480	#else
7481	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7482	{
7483	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7484	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7485	}
7486	else
7487	{
7488	/*
7489	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
7490	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7491	* re-construct CR0. See @bugref{9180#c95} for details.
7492	*/
7493	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7494	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7495	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7496	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
7497	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
7498	}
7499	#endif
7500	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7501	CPUMSetGuestCR0(pVCpu, u64Cr0);
7502	VMMRZCallRing3Enable(pVCpu);
7503	}
7504
7505	if (fWhat & CPUMCTX_EXTRN_CR4)
7506	{
7507	uint64_t u64Cr4;
7508	uint64_t u64Shadow;
7509	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
7510	rc \|= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
7511	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7512	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7513	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7514	#else
7515	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7516	{
7517	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7518	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7519	}
7520	else
7521	{
7522	/*
7523	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
7524	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7525	* re-construct CR4. See @bugref{9180#c95} for details.
7526	*/
7527	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7528	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7529	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7530	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
7531	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
7532	}
7533	#endif
7534	pCtx->cr4 = u64Cr4;
7535	}
7536
7537	if (fWhat & CPUMCTX_EXTRN_CR3)
7538	{
7539	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7540	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7541	\|\| ( pVM->hm.s.fNestedPaging
7542	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7543	{
7544	uint64_t u64Cr3;
7545	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
7546	if (pCtx->cr3 != u64Cr3)
7547	{
7548	pCtx->cr3 = u64Cr3;
7549	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7550	}
7551
7552	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7553	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7554	if (CPUMIsGuestInPAEModeEx(pCtx))
7555	{
7556	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
7557	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
7558	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
7559	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
7560	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7561	}
7562	}
7563	}
7564	}
7565
7566	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7567	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7568	{
7569	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7570	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7571	{
7572	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7573	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7574	if (RT_SUCCESS(rc))
7575	{ /* likely */ }
7576	else
7577	break;
7578	}
7579	}
7580	#endif
7581	} while (0);
7582
7583	if (RT_SUCCESS(rc))
7584	{
7585	/* Update fExtrn. */
7586	pCtx->fExtrn &= ~fWhat;
7587
7588	/* If everything has been imported, clear the HM keeper bit. */
7589	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7590	{
7591	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7592	Assert(!pCtx->fExtrn);
7593	}
7594	}
7595	}
7596	else
7597	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7598
7599	/*
7600	* Restore interrupts.
7601	*/
7602	ASMSetFlags(fEFlags);
7603
7604	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7605
7606	if (RT_SUCCESS(rc))
7607	{ /* likely */ }
7608	else
7609	return rc;
7610
7611	/*
7612	* Honor any pending CR3 updates.
7613	*
7614	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
7615	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7616	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7617	*
7618	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7619	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7620	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7621	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7622	*
7623	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7624	*/
7625	if (VMMRZCallRing3IsEnabled(pVCpu))
7626	{
7627	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7628	{
7629	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7630	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7631	}
7632
7633	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7634	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7635
7636	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7637	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7638	}
7639
7640	return VINF_SUCCESS;
7641	}
7642
7643
7644	/**
7645	* Saves the guest state from the VMCS into the guest-CPU context.
7646	*
7647	* @returns VBox status code.
7648	* @param pVCpu The cross context virtual CPU structure.
7649	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7650	*/
7651	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
7652	{
7653	AssertPtr(pVCpu);
7654	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7655	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7656	}
7657
7658
7659	/**
7660	* Check per-VM and per-VCPU force flag actions that require us to go back to
7661	* ring-3 for one reason or another.
7662	*
7663	* @returns Strict VBox status code (i.e. informational status codes too)
7664	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7665	* ring-3.
7666	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7667	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7668	* interrupts)
7669	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7670	* all EMTs to be in ring-3.
7671	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7672	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7673	* to the EM loop.
7674	*
7675	* @param pVCpu The cross context virtual CPU structure.
7676	* @param fStepping Whether we are single-stepping the guest using the
7677	* hypervisor debugger.
7678	*
7679	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7680	* is no longer in VMX non-root mode.
7681	*/
7682	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPUCC pVCpu, bool fStepping)
7683	{
7684	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7685
7686	/*
7687	* Update pending interrupts into the APIC's IRR.
7688	*/
7689	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7690	APICUpdatePendingInterrupts(pVCpu);
7691
7692	/*
7693	* Anything pending? Should be more likely than not if we're doing a good job.
7694	*/
7695	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7696	if ( !fStepping
7697	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7698	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7699	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7700	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7701	return VINF_SUCCESS;
7702
7703	/* Pending PGM C3 sync. */
7704	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7705	{
7706	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7707	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7708	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7709	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7710	if (rcStrict != VINF_SUCCESS)
7711	{
7712	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7713	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7714	return rcStrict;
7715	}
7716	}
7717
7718	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7719	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7720	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7721	{
7722	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7723	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7724	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7725	return rc;
7726	}
7727
7728	/* Pending VM request packets, such as hardware interrupts. */
7729	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7730	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7731	{
7732	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7733	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7734	return VINF_EM_PENDING_REQUEST;
7735	}
7736
7737	/* Pending PGM pool flushes. */
7738	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7739	{
7740	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7741	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7742	return VINF_PGM_POOL_FLUSH_PENDING;
7743	}
7744
7745	/* Pending DMA requests. */
7746	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7747	{
7748	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7749	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7750	return VINF_EM_RAW_TO_R3;
7751	}
7752
7753	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7754	/* Pending nested-guest APIC-write (has highest priority among nested-guest FFs). */
7755	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7756	{
7757	Log4Func(("Pending nested-guest APIC-write\n"));
7758	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
7759	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7760	return rcStrict;
7761	}
7762	/** @todo VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_PREEMPT_TIMER */
7763	#endif
7764
7765	return VINF_SUCCESS;
7766	}
7767
7768
7769	/**
7770	* Converts any TRPM trap into a pending HM event. This is typically used when
7771	* entering from ring-3 (not longjmp returns).
7772	*
7773	* @param pVCpu The cross context virtual CPU structure.
7774	*/
7775	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
7776	{
7777	Assert(TRPMHasTrap(pVCpu));
7778	Assert(!pVCpu->hm.s.Event.fPending);
7779
7780	uint8_t uVector;
7781	TRPMEVENT enmTrpmEvent;
7782	uint32_t uErrCode;
7783	RTGCUINTPTR GCPtrFaultAddress;
7784	uint8_t cbInstr;
7785	bool fIcebp;
7786
7787	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
7788	AssertRC(rc);
7789
7790	uint32_t u32IntInfo;
7791	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
7792	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
7793
7794	rc = TRPMResetTrap(pVCpu);
7795	AssertRC(rc);
7796	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7797	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7798
7799	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7800	}
7801
7802
7803	/**
7804	* Converts the pending HM event into a TRPM trap.
7805	*
7806	* @param pVCpu The cross context virtual CPU structure.
7807	*/
7808	static void hmR0VmxPendingEventToTrpmTrap(PVMCPUCC pVCpu)
7809	{
7810	Assert(pVCpu->hm.s.Event.fPending);
7811
7812	/* If a trap was already pending, we did something wrong! */
7813	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7814
7815	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
7816	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
7817	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
7818
7819	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7820
7821	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7822	AssertRC(rc);
7823
7824	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7825	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
7826
7827	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
7828	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7829	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
7830	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7831
7832	if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7833	TRPMSetTrapDueToIcebp(pVCpu);
7834
7835	/* We're now done converting the pending event. */
7836	pVCpu->hm.s.Event.fPending = false;
7837	}
7838
7839
7840	/**
7841	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7842	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7843	*
7844	* @param pVCpu The cross context virtual CPU structure.
7845	* @param pVmcsInfo The VMCS info. object.
7846	*/
7847	static void hmR0VmxSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7848	{
7849	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7850	{
7851	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7852	{
7853	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7854	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7855	AssertRC(rc);
7856	}
7857	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7858	}
7859
7860
7861	/**
7862	* Clears the interrupt-window exiting control in the VMCS.
7863	*
7864	* @param pVmcsInfo The VMCS info. object.
7865	*/
7866	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7867	{
7868	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7869	{
7870	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7871	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7872	AssertRC(rc);
7873	}
7874	}
7875
7876
7877	/**
7878	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7879	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7880	*
7881	* @param pVCpu The cross context virtual CPU structure.
7882	* @param pVmcsInfo The VMCS info. object.
7883	*/
7884	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7885	{
7886	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7887	{
7888	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7889	{
7890	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7891	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7892	AssertRC(rc);
7893	Log4Func(("Setup NMI-window exiting\n"));
7894	}
7895	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7896	}
7897
7898
7899	/**
7900	* Clears the NMI-window exiting control in the VMCS.
7901	*
7902	* @param pVmcsInfo The VMCS info. object.
7903	*/
7904	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7905	{
7906	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7907	{
7908	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7909	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7910	AssertRC(rc);
7911	}
7912	}
7913
7914
7915	/**
7916	* Does the necessary state syncing before returning to ring-3 for any reason
7917	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7918	*
7919	* @returns VBox status code.
7920	* @param pVCpu The cross context virtual CPU structure.
7921	* @param fImportState Whether to import the guest state from the VMCS back
7922	* to the guest-CPU context.
7923	*
7924	* @remarks No-long-jmp zone!!!
7925	*/
7926	static int hmR0VmxLeave(PVMCPUCC pVCpu, bool fImportState)
7927	{
7928	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7929	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7930
7931	RTCPUID const idCpu = RTMpCpuId();
7932	Log4Func(("HostCpuId=%u\n", idCpu));
7933
7934	/*
7935	* !!! IMPORTANT !!!
7936	* If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
7937	*/
7938
7939	/* Save the guest state if necessary. */
7940	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7941	if (fImportState)
7942	{
7943	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
7944	AssertRCReturn(rc, rc);
7945	}
7946
7947	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7948	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7949	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7950
7951	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7952	#ifdef VBOX_STRICT
7953	if (CPUMIsHyperDebugStateActive(pVCpu))
7954	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7955	#endif
7956	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7957	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7958	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7959
7960	/* Restore host-state bits that VT-x only restores partially. */
7961	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7962	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7963	{
7964	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7965	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7966	}
7967	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7968
7969	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7970	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7971	{
7972	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7973	if (!fImportState)
7974	{
7975	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
7976	AssertRCReturn(rc, rc);
7977	}
7978	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7979	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7980	}
7981	else
7982	pVCpu->hm.s.vmx.fLazyMsrs = 0;
7983
7984	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7985	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
7986
7987	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7988	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7989	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7990	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7991	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7992	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7993	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7994	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7995	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
7996	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7997
7998	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7999
8000	/** @todo This partially defeats the purpose of having preemption hooks.
8001	* The problem is, deregistering the hooks should be moved to a place that
8002	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8003	* context.
8004	*/
8005	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8006	AssertRCReturn(rc, rc);
8007
8008	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8009	/*
8010	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8011	* clear a shadow VMCS before allowing that VMCS to become active on another
8012	* logical processor. We may or may not be importing guest state which clears
8013	* it, so cover for it here.
8014	*
8015	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8016	*/
8017	if ( pVmcsInfo->pvShadowVmcs
8018	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8019	{
8020	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8021	AssertRCReturn(rc, rc);
8022	}
8023
8024	/*
8025	* Flag that we need to re-export the host state if we switch to this VMCS before
8026	* executing guest or nested-guest code.
8027	*/
8028	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
8029	#endif
8030
8031	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8032	NOREF(idCpu);
8033	return VINF_SUCCESS;
8034	}
8035
8036
8037	/**
8038	* Leaves the VT-x session.
8039	*
8040	* @returns VBox status code.
8041	* @param pVCpu The cross context virtual CPU structure.
8042	*
8043	* @remarks No-long-jmp zone!!!
8044	*/
8045	static int hmR0VmxLeaveSession(PVMCPUCC pVCpu)
8046	{
8047	HM_DISABLE_PREEMPT(pVCpu);
8048	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8049	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8051
8052	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8053	and done this from the VMXR0ThreadCtxCallback(). */
8054	if (!pVCpu->hm.s.fLeaveDone)
8055	{
8056	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8057	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8058	pVCpu->hm.s.fLeaveDone = true;
8059	}
8060	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8061
8062	/*
8063	* !!! IMPORTANT !!!
8064	* If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
8065	*/
8066
8067	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8068	/** @todo Deregistering here means we need to VMCLEAR always
8069	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8070	* for calling VMMR0ThreadCtxHookDisable here! */
8071	VMMR0ThreadCtxHookDisable(pVCpu);
8072
8073	/* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
8074	int rc = HMR0LeaveCpu(pVCpu);
8075	HM_RESTORE_PREEMPT();
8076	return rc;
8077	}
8078
8079
8080	/**
8081	* Does the necessary state syncing before doing a longjmp to ring-3.
8082	*
8083	* @returns VBox status code.
8084	* @param pVCpu The cross context virtual CPU structure.
8085	*
8086	* @remarks No-long-jmp zone!!!
8087	*/
8088	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPUCC pVCpu)
8089	{
8090	return hmR0VmxLeaveSession(pVCpu);
8091	}
8092
8093
8094	/**
8095	* Take necessary actions before going back to ring-3.
8096	*
8097	* An action requires us to go back to ring-3. This function does the necessary
8098	* steps before we can safely return to ring-3. This is not the same as longjmps
8099	* to ring-3, this is voluntary and prepares the guest so it may continue
8100	* executing outside HM (recompiler/IEM).
8101	*
8102	* @returns VBox status code.
8103	* @param pVCpu The cross context virtual CPU structure.
8104	* @param rcExit The reason for exiting to ring-3. Can be
8105	* VINF_VMM_UNKNOWN_RING3_CALL.
8106	*/
8107	static int hmR0VmxExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
8108	{
8109	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8110
8111	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8112	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8113	{
8114	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8115	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8116	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8117	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8118	}
8119
8120	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8121	VMMRZCallRing3Disable(pVCpu);
8122	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8123
8124	/*
8125	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8126	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8127	*
8128	* This is because execution may continue from ring-3 and we would need to inject
8129	* the event from there (hence place it back in TRPM).
8130	*/
8131	if (pVCpu->hm.s.Event.fPending)
8132	{
8133	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8134	Assert(!pVCpu->hm.s.Event.fPending);
8135
8136	/* Clear the events from the VMCS. */
8137	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8138	AssertRC(rc);
8139	}
8140	#ifdef VBOX_STRICT
8141	else
8142	{
8143	/*
8144	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8145	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8146	* occasionally, see @bugref{9180#c42}.
8147	*
8148	* However, if the VM-entry failed, any VM entry-interruption info. field would
8149	* be left unmodified as the event would not have been injected to the guest. In
8150	* such cases, don't assert, we're not going to continue guest execution anyway.
8151	*/
8152	uint32_t uExitReason;
8153	uint32_t uEntryIntInfo;
8154	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8155	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8156	AssertRC(rc);
8157	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8158	}
8159	#endif
8160
8161	/*
8162	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8163	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8164	* (e.g. TPR below threshold).
8165	*/
8166	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8167	{
8168	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8169	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8170	}
8171
8172	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8173	and if we're injecting an event we should have a TRPM trap pending. */
8174	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8175	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8176	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8177	#endif
8178
8179	/* Save guest state and restore host state bits. */
8180	int rc = hmR0VmxLeaveSession(pVCpu);
8181	AssertRCReturn(rc, rc);
8182	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8183
8184	/* Thread-context hooks are unregistered at this point!!! */
8185	/* Ring-3 callback notifications are unregistered at this point!!! */
8186
8187	/* Sync recompiler state. */
8188	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8189	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8190	\| CPUM_CHANGED_LDTR
8191	\| CPUM_CHANGED_GDTR
8192	\| CPUM_CHANGED_IDTR
8193	\| CPUM_CHANGED_TR
8194	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8195	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8196	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8197	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8198
8199	Assert(!pVCpu->hm.s.fClearTrapFlag);
8200
8201	/* Update the exit-to-ring 3 reason. */
8202	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8203
8204	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8205	if ( rcExit != VINF_EM_RAW_INTERRUPT
8206	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8207	{
8208	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8209	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8210	}
8211
8212	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8213	VMMRZCallRing3Enable(pVCpu);
8214	return rc;
8215	}
8216
8217
8218	/**
8219	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8220	* longjump to ring-3 and possibly get preempted.
8221	*
8222	* @returns VBox status code.
8223	* @param pVCpu The cross context virtual CPU structure.
8224	* @param enmOperation The operation causing the ring-3 longjump.
8225	*/
8226	VMMR0DECL(int) VMXR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
8227	{
8228	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8229	{
8230	/*
8231	* !!! IMPORTANT !!!
8232	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8233	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8234	*/
8235	VMMRZCallRing3RemoveNotification(pVCpu);
8236	VMMRZCallRing3Disable(pVCpu);
8237	HM_DISABLE_PREEMPT(pVCpu);
8238
8239	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8240	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8241	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8242	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8243
8244	/* Restore host-state bits that VT-x only restores partially. */
8245	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8246	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8247	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8248	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8249
8250	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8251	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8252	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8253
8254	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8255	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8256	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8257
8258	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8259	cleared as part of importing the guest state above. */
8260	hmR0VmxClearVmcs(pVmcsInfo);
8261
8262	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8263	VMMR0ThreadCtxHookDisable(pVCpu);
8264
8265	/* Leave HM context. This takes care of local init (term). */
8266	HMR0LeaveCpu(pVCpu);
8267	HM_RESTORE_PREEMPT();
8268	return VINF_SUCCESS;
8269	}
8270
8271	Assert(pVCpu);
8272	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8273	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8274
8275	VMMRZCallRing3Disable(pVCpu);
8276	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8277
8278	Log4Func(("-> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8279
8280	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8281	AssertRCReturn(rc, rc);
8282
8283	VMMRZCallRing3Enable(pVCpu);
8284	return VINF_SUCCESS;
8285	}
8286
8287
8288	/**
8289	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8290	* stack.
8291	*
8292	* @returns Strict VBox status code (i.e. informational status codes too).
8293	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8294	* @param pVCpu The cross context virtual CPU structure.
8295	* @param uValue The value to push to the guest stack.
8296	*/
8297	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
8298	{
8299	/*
8300	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8301	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8302	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8303	*/
8304	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8305	if (pCtx->sp == 1)
8306	return VINF_EM_RESET;
8307	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8308	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8309	AssertRC(rc);
8310	return rc;
8311	}
8312
8313
8314	/**
8315	* Injects an event into the guest upon VM-entry by updating the relevant fields
8316	* in the VM-entry area in the VMCS.
8317	*
8318	* @returns Strict VBox status code (i.e. informational status codes too).
8319	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8320	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8321	*
8322	* @param pVCpu The cross context virtual CPU structure.
8323	* @param pVmxTransient The VMX-transient structure.
8324	* @param pEvent The event being injected.
8325	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8326	* will be updated if necessary. This cannot not be NULL.
8327	* @param fStepping Whether we're single-stepping guest execution and should
8328	* return VINF_EM_DBG_STEPPED if the event is injected
8329	* directly (registers modified by us, not by hardware on
8330	* VM-entry).
8331	*/
8332	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8333	uint32_t *pfIntrState)
8334	{
8335	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8336	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8337	Assert(pfIntrState);
8338
8339	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8340	uint32_t u32IntInfo = pEvent->u64IntInfo;
8341	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8342	uint32_t const cbInstr = pEvent->cbInstr;
8343	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8344	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8345	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8346
8347	#ifdef VBOX_STRICT
8348	/*
8349	* Validate the error-code-valid bit for hardware exceptions.
8350	* No error codes for exceptions in real-mode.
8351	*
8352	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8353	*/
8354	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8355	&& !CPUMIsGuestInRealModeEx(pCtx))
8356	{
8357	switch (uVector)
8358	{
8359	case X86_XCPT_PF:
8360	case X86_XCPT_DF:
8361	case X86_XCPT_TS:
8362	case X86_XCPT_NP:
8363	case X86_XCPT_SS:
8364	case X86_XCPT_GP:
8365	case X86_XCPT_AC:
8366	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8367	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8368	RT_FALL_THRU();
8369	default:
8370	break;
8371	}
8372	}
8373
8374	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8375	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8376	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8377	#endif
8378
8379	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8380
8381	/*
8382	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8383	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8384	* interrupt handler in the (real-mode) guest.
8385	*
8386	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8387	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8388	*/
8389	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8390	{
8391	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8392	{
8393	/*
8394	* For CPUs with unrestricted guest execution enabled and with the guest
8395	* in real-mode, we must not set the deliver-error-code bit.
8396	*
8397	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8398	*/
8399	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8400	}
8401	else
8402	{
8403	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8404	Assert(PDMVmmDevHeapIsEnabled(pVM));
8405	Assert(pVM->hm.s.vmx.pRealModeTSS);
8406	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8407
8408	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8409	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8410	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8411	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8412	AssertRCReturn(rc2, rc2);
8413
8414	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8415	size_t const cbIdtEntry = sizeof(X86IDTR16);
8416	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8417	{
8418	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8419	if (uVector == X86_XCPT_DF)
8420	return VINF_EM_RESET;
8421
8422	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8423	No error codes for exceptions in real-mode. */
8424	if (uVector == X86_XCPT_GP)
8425	{
8426	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8427	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8428	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8429	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8430	HMEVENT EventXcptDf;
8431	RT_ZERO(EventXcptDf);
8432	EventXcptDf.u64IntInfo = uXcptDfInfo;
8433	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8434	}
8435
8436	/*
8437	* If we're injecting an event with no valid IDT entry, inject a #GP.
8438	* No error codes for exceptions in real-mode.
8439	*
8440	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8441	*/
8442	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8443	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8444	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8445	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8446	HMEVENT EventXcptGp;
8447	RT_ZERO(EventXcptGp);
8448	EventXcptGp.u64IntInfo = uXcptGpInfo;
8449	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8450	}
8451
8452	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8453	uint16_t uGuestIp = pCtx->ip;
8454	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8455	{
8456	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8457	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8458	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8459	}
8460	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8461	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8462
8463	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8464	X86IDTR16 IdtEntry;
8465	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8466	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8467	AssertRCReturn(rc2, rc2);
8468
8469	/* Construct the stack frame for the interrupt/exception handler. */
8470	VBOXSTRICTRC rcStrict;
8471	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8472	if (rcStrict == VINF_SUCCESS)
8473	{
8474	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8475	if (rcStrict == VINF_SUCCESS)
8476	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8477	}
8478
8479	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8480	if (rcStrict == VINF_SUCCESS)
8481	{
8482	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8483	pCtx->rip = IdtEntry.offSel;
8484	pCtx->cs.Sel = IdtEntry.uSel;
8485	pCtx->cs.ValidSel = IdtEntry.uSel;
8486	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8487	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8488	&& uVector == X86_XCPT_PF)
8489	pCtx->cr2 = GCPtrFault;
8490
8491	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8492	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8493	\| HM_CHANGED_GUEST_RSP);
8494
8495	/*
8496	* If we delivered a hardware exception (other than an NMI) and if there was
8497	* block-by-STI in effect, we should clear it.
8498	*/
8499	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8500	{
8501	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8502	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8503	Log4Func(("Clearing inhibition due to STI\n"));
8504	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8505	}
8506
8507	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8508	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8509
8510	/*
8511	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8512	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8513	*/
8514	pVCpu->hm.s.Event.fPending = false;
8515
8516	/*
8517	* If we eventually support nested-guest execution without unrestricted guest execution,
8518	* we should set fInterceptEvents here.
8519	*/
8520	Assert(!pVmxTransient->fIsNestedGuest);
8521
8522	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8523	if (fStepping)
8524	rcStrict = VINF_EM_DBG_STEPPED;
8525	}
8526	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8527	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8528	return rcStrict;
8529	}
8530	}
8531
8532	/*
8533	* Validate.
8534	*/
8535	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8536	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8537
8538	/*
8539	* Inject the event into the VMCS.
8540	*/
8541	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8542	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8543	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8544	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8545	AssertRC(rc);
8546
8547	/*
8548	* Update guest CR2 if this is a page-fault.
8549	*/
8550	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8551	pCtx->cr2 = GCPtrFault;
8552
8553	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8554	return VINF_SUCCESS;
8555	}
8556
8557
8558	/**
8559	* Evaluates the event to be delivered to the guest and sets it as the pending
8560	* event.
8561	*
8562	* @returns Strict VBox status code (i.e. informational status codes too).
8563	* @param pVCpu The cross context virtual CPU structure.
8564	* @param pVmxTransient The VMX-transient structure.
8565	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8566	*/
8567	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8568	{
8569	Assert(pfIntrState);
8570	Assert(!TRPMHasTrap(pVCpu));
8571
8572	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8573	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8574	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8575
8576	/*
8577	* Get the current interruptibility-state of the guest or nested-guest and
8578	* then figure out what needs to be injected.
8579	*/
8580	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8581	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8582	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8583	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8584
8585	/* We don't support block-by-SMI yet.*/
8586	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
8587
8588	/* Block-by-STI must not be set when interrupts are disabled. */
8589	if (fBlockSti)
8590	{
8591	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
8592	Assert(pCtx->eflags.Bits.u1IF);
8593	}
8594
8595	/* Update interruptibility state to the caller. */
8596	*pfIntrState = fIntrState;
8597
8598	/*
8599	* Toggling of interrupt force-flags here is safe since we update TRPM on
8600	* premature exits to ring-3 before executing guest code, see hmR0VmxExitToRing3().
8601	* We must NOT restore these force-flags.
8602	*/
8603
8604	/** @todo SMI. SMIs take priority over NMIs. */
8605
8606	/*
8607	* Check if an NMI is pending and if the guest or nested-guest can receive them.
8608	* NMIs take priority over external interrupts.
8609	*/
8610	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8611	{
8612	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8613	if ( !pVCpu->hm.s.Event.fPending
8614	&& !fBlockNmi
8615	&& !fBlockSti
8616	&& !fBlockMovSS)
8617	{
8618	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8619	if ( fIsNestedGuest
8620	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8621	return IEMExecVmxVmexitXcptNmi(pVCpu);
8622	#endif
8623	hmR0VmxSetPendingXcptNmi(pVCpu);
8624	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8625	Log4Func(("Pending NMI\n"));
8626	}
8627	else if (!fIsNestedGuest)
8628	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8629	/* else: for nested-guests, NMI-window exiting will be picked up when merging VMCS controls. */
8630	}
8631	/*
8632	* Check if an external interrupt (PIC/APIC) is pending and if the guest or nested-guest
8633	* can receive them. Once PDMGetInterrupt() returns a valid interrupt we -must- deliver
8634	* the interrupt. We can no longer re-request it from the APIC.
8635	*/
8636	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8637	&& !pVCpu->hm.s.fSingleInstruction)
8638	{
8639	Assert(!DBGFIsStepping(pVCpu));
8640	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8641	AssertRCReturn(rc, rc);
8642
8643	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8644	if ( !pVCpu->hm.s.Event.fPending
8645	&& !fBlockInt
8646	&& !fBlockSti
8647	&& !fBlockMovSS)
8648	{
8649	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8650	if ( fIsNestedGuest
8651	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8652	&& !CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8653	{
8654	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8655	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8656	return rcStrict;
8657	}
8658	#endif
8659	uint8_t u8Interrupt;
8660	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8661	if (RT_SUCCESS(rc))
8662	{
8663	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8664	if ( fIsNestedGuest
8665	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8666	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8667	{
8668	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8669	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8670	return rcStrict;
8671	}
8672	#endif
8673	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8674	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8675	}
8676	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8677	{
8678	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8679
8680	if ( !fIsNestedGuest
8681	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8682	hmR0VmxApicSetTprThreshold(pVmcsInfo, u8Interrupt >> 4);
8683	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8684
8685	/*
8686	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8687	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8688	* need to re-set this force-flag here.
8689	*/
8690	}
8691	else
8692	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8693	}
8694	else if (!fIsNestedGuest)
8695	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8696	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCS controls. */
8697	}
8698
8699	return VINF_SUCCESS;
8700	}
8701
8702
8703	/**
8704	* Injects any pending events into the guest if the guest is in a state to
8705	* receive them.
8706	*
8707	* @returns Strict VBox status code (i.e. informational status codes too).
8708	* @param pVCpu The cross context virtual CPU structure.
8709	* @param pVmxTransient The VMX-transient structure.
8710	* @param fIntrState The VT-x guest-interruptibility state.
8711	* @param fStepping Whether we are single-stepping the guest using the
8712	* hypervisor debugger and should return
8713	* VINF_EM_DBG_STEPPED if the event was dispatched
8714	* directly.
8715	*/
8716	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8717	{
8718	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8719	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8720
8721	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8722	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8723
8724	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8725	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8726	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8727	Assert(!TRPMHasTrap(pVCpu));
8728
8729	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8730	if (pVCpu->hm.s.Event.fPending)
8731	{
8732	/*
8733	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8734	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8735	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8736	*
8737	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8738	*/
8739	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8740	#ifdef VBOX_STRICT
8741	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8742	{
8743	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8744	Assert(!fBlockInt);
8745	Assert(!fBlockSti);
8746	Assert(!fBlockMovSS);
8747	}
8748	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8749	{
8750	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8751	Assert(!fBlockSti);
8752	Assert(!fBlockMovSS);
8753	Assert(!fBlockNmi);
8754	}
8755	#endif
8756	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8757	uIntType));
8758
8759	/*
8760	* Inject the event and get any changes to the guest-interruptibility state.
8761	*
8762	* The guest-interruptibility state may need to be updated if we inject the event
8763	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8764	*/
8765	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8766	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8767
8768	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8769	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8770	else
8771	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8772	}
8773
8774	/*
8775	* Update the guest-interruptibility state.
8776	*
8777	* This is required for the real-on-v86 software interrupt injection case above, as well as
8778	* updates to the guest state from ring-3 or IEM/REM.
8779	*/
8780	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8781	AssertRC(rc);
8782
8783	/*
8784	* There's no need to clear the VM-entry interruption-information field here if we're not
8785	* injecting anything. VT-x clears the valid bit on every VM-exit.
8786	*
8787	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8788	*/
8789
8790	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8791	NOREF(fBlockMovSS); NOREF(fBlockSti);
8792	return rcStrict;
8793	}
8794
8795
8796	/**
8797	* Enters the VT-x session.
8798	*
8799	* @returns VBox status code.
8800	* @param pVCpu The cross context virtual CPU structure.
8801	*/
8802	VMMR0DECL(int) VMXR0Enter(PVMCPUCC pVCpu)
8803	{
8804	AssertPtr(pVCpu);
8805	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8806	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8807
8808	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8809	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8810	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8811
8812	#ifdef VBOX_STRICT
8813	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8814	RTCCUINTREG uHostCr4 = ASMGetCR4();
8815	if (!(uHostCr4 & X86_CR4_VMXE))
8816	{
8817	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8818	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8819	}
8820	#endif
8821
8822	/*
8823	* Load the appropriate VMCS as the current and active one.
8824	*/
8825	PVMXVMCSINFO pVmcsInfo;
8826	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8827	if (!fInNestedGuestMode)
8828	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8829	else
8830	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8831	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8832	if (RT_SUCCESS(rc))
8833	{
8834	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8835	pVCpu->hm.s.fLeaveDone = false;
8836	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8837
8838	/*
8839	* Do the EMT scheduled L1D flush here if needed.
8840	*/
8841	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8842	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8843	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8844	hmR0MdsClear();
8845	}
8846	return rc;
8847	}
8848
8849
8850	/**
8851	* The thread-context callback (only on platforms which support it).
8852	*
8853	* @param enmEvent The thread-context event.
8854	* @param pVCpu The cross context virtual CPU structure.
8855	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8856	* @thread EMT(pVCpu)
8857	*/
8858	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
8859	{
8860	AssertPtr(pVCpu);
8861	RT_NOREF1(fGlobalInit);
8862
8863	switch (enmEvent)
8864	{
8865	case RTTHREADCTXEVENT_OUT:
8866	{
8867	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8868	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8869	VMCPU_ASSERT_EMT(pVCpu);
8870
8871	/* No longjmps (logger flushes, locks) in this fragile context. */
8872	VMMRZCallRing3Disable(pVCpu);
8873	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8874
8875	/* Restore host-state (FPU, debug etc.) */
8876	if (!pVCpu->hm.s.fLeaveDone)
8877	{
8878	/*
8879	* Do -not- import the guest-state here as we might already be in the middle of importing
8880	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8881	*/
8882	hmR0VmxLeave(pVCpu, false /* fImportState */);
8883	pVCpu->hm.s.fLeaveDone = true;
8884	}
8885
8886	/* Leave HM context, takes care of local init (term). */
8887	int rc = HMR0LeaveCpu(pVCpu);
8888	AssertRC(rc);
8889
8890	/* Restore longjmp state. */
8891	VMMRZCallRing3Enable(pVCpu);
8892	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8893	break;
8894	}
8895
8896	case RTTHREADCTXEVENT_IN:
8897	{
8898	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8899	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8900	VMCPU_ASSERT_EMT(pVCpu);
8901
8902	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8903	VMMRZCallRing3Disable(pVCpu);
8904	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8905
8906	/* Initialize the bare minimum state required for HM. This takes care of
8907	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8908	int rc = hmR0EnterCpu(pVCpu);
8909	AssertRC(rc);
8910	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8911	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8912
8913	/* Load the active VMCS as the current one. */
8914	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8915	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8916	AssertRC(rc);
8917	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8918	pVCpu->hm.s.fLeaveDone = false;
8919
8920	/* Do the EMT scheduled L1D flush if needed. */
8921	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8922	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8923
8924	/* Restore longjmp state. */
8925	VMMRZCallRing3Enable(pVCpu);
8926	break;
8927	}
8928
8929	default:
8930	break;
8931	}
8932	}
8933
8934
8935	/**
8936	* Exports the host state into the VMCS host-state area.
8937	* Sets up the VM-exit MSR-load area.
8938	*
8939	* The CPU state will be loaded from these fields on every successful VM-exit.
8940	*
8941	* @returns VBox status code.
8942	* @param pVCpu The cross context virtual CPU structure.
8943	*
8944	* @remarks No-long-jump zone!!!
8945	*/
8946	static int hmR0VmxExportHostState(PVMCPUCC pVCpu)
8947	{
8948	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8949
8950	int rc = VINF_SUCCESS;
8951	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8952	{
8953	hmR0VmxExportHostControlRegs();
8954
8955	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8956	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8957
8958	hmR0VmxExportHostMsrs(pVCpu);
8959
8960	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8961	}
8962	return rc;
8963	}
8964
8965
8966	/**
8967	* Saves the host state in the VMCS host-state.
8968	*
8969	* @returns VBox status code.
8970	* @param pVCpu The cross context virtual CPU structure.
8971	*
8972	* @remarks No-long-jump zone!!!
8973	*/
8974	VMMR0DECL(int) VMXR0ExportHostState(PVMCPUCC pVCpu)
8975	{
8976	AssertPtr(pVCpu);
8977	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8978
8979	/*
8980	* Export the host state here while entering HM context.
8981	* When thread-context hooks are used, we might get preempted and have to re-save the host
8982	* state but most of the time we won't be, so do it here before we disable interrupts.
8983	*/
8984	return hmR0VmxExportHostState(pVCpu);
8985	}
8986
8987
8988	/**
8989	* Exports the guest state into the VMCS guest-state area.
8990	*
8991	* The will typically be done before VM-entry when the guest-CPU state and the
8992	* VMCS state may potentially be out of sync.
8993	*
8994	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8995	* VM-entry controls.
8996	* Sets up the appropriate VMX non-root function to execute guest code based on
8997	* the guest CPU mode.
8998	*
8999	* @returns VBox strict status code.
9000	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9001	* without unrestricted guest execution and the VMMDev is not presently
9002	* mapped (e.g. EFI32).
9003	*
9004	* @param pVCpu The cross context virtual CPU structure.
9005	* @param pVmxTransient The VMX-transient structure.
9006	*
9007	* @remarks No-long-jump zone!!!
9008	*/
9009	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9010	{
9011	AssertPtr(pVCpu);
9012	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9013	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9014
9015	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9016
9017	/*
9018	* Determine real-on-v86 mode.
9019	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9020	*/
9021	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9022	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9023	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9024	pVmcsInfo->RealMode.fRealOnV86Active = false;
9025	else
9026	{
9027	Assert(!pVmxTransient->fIsNestedGuest);
9028	pVmcsInfo->RealMode.fRealOnV86Active = true;
9029	}
9030
9031	/*
9032	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9033	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9034	*/
9035	int rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9036	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9037
9038	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9039	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9040
9041	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9042	if (rcStrict == VINF_SUCCESS)
9043	{ /* likely */ }
9044	else
9045	{
9046	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9047	return rcStrict;
9048	}
9049
9050	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9051	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9052
9053	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9054	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9055
9056	hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9057	hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9058	hmR0VmxExportGuestRip(pVCpu);
9059	hmR0VmxExportGuestRsp(pVCpu);
9060	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9061
9062	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9063	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9064
9065	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9066	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9067	\| HM_CHANGED_GUEST_CR2
9068	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9069	\| HM_CHANGED_GUEST_X87
9070	\| HM_CHANGED_GUEST_SSE_AVX
9071	\| HM_CHANGED_GUEST_OTHER_XSAVE
9072	\| HM_CHANGED_GUEST_XCRx
9073	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9074	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9075	\| HM_CHANGED_GUEST_TSC_AUX
9076	\| HM_CHANGED_GUEST_OTHER_MSRS
9077	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9078
9079	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9080	return rc;
9081	}
9082
9083
9084	/**
9085	* Exports the state shared between the host and guest into the VMCS.
9086	*
9087	* @param pVCpu The cross context virtual CPU structure.
9088	* @param pVmxTransient The VMX-transient structure.
9089	*
9090	* @remarks No-long-jump zone!!!
9091	*/
9092	static void hmR0VmxExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9093	{
9094	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9095	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9096
9097	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9098	{
9099	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9100	AssertRC(rc);
9101	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9102
9103	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9104	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9105	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9106	}
9107
9108	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9109	{
9110	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9111	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9112	}
9113
9114	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9115	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9116	}
9117
9118
9119	/**
9120	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9121	*
9122	* @returns Strict VBox status code (i.e. informational status codes too).
9123	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9124	* without unrestricted guest execution and the VMMDev is not presently
9125	* mapped (e.g. EFI32).
9126	*
9127	* @param pVCpu The cross context virtual CPU structure.
9128	* @param pVmxTransient The VMX-transient structure.
9129	*
9130	* @remarks No-long-jump zone!!!
9131	*/
9132	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9133	{
9134	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9135	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9136	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9137
9138	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9139	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9140	#endif
9141
9142	/*
9143	* For many VM-exits only RIP/RSP/RFLAGS (and HWVIRT state when executing a nested-guest)
9144	* changes. First try to export only these without going through all other changed-flag checks.
9145	*/
9146	VBOXSTRICTRC rcStrict;
9147	uint64_t const fCtxMask = HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
9148	uint64_t const fMinimalMask = HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT;
9149	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9150
9151	/* If only RIP/RSP/RFLAGS/HWVIRT changed, export only those (quicker, happens more often).*/
9152	if ( (fCtxChanged & fMinimalMask)
9153	&& !(fCtxChanged & (fCtxMask & ~fMinimalMask)))
9154	{
9155	hmR0VmxExportGuestRip(pVCpu);
9156	hmR0VmxExportGuestRsp(pVCpu);
9157	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9158	rcStrict = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9159	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9160	}
9161	/* If anything else also changed, go through the full export routine and export as required. */
9162	else if (fCtxChanged & fCtxMask)
9163	{
9164	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9165	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9166	{ /* likely */}
9167	else
9168	{
9169	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9170	VBOXSTRICTRC_VAL(rcStrict)));
9171	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9172	return rcStrict;
9173	}
9174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9175	}
9176	/* Nothing changed, nothing to load here. */
9177	else
9178	rcStrict = VINF_SUCCESS;
9179
9180	#ifdef VBOX_STRICT
9181	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9182	uint64_t const fCtxChangedCur = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9183	AssertMsg(!(fCtxChangedCur & fCtxMask), ("fCtxChangedCur=%#RX64\n", fCtxChangedCur));
9184	#endif
9185	return rcStrict;
9186	}
9187
9188
9189	/**
9190	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9191	* and update error record fields accordingly.
9192	*
9193	* @returns VMX_IGS_* error codes.
9194	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9195	* wrong with the guest state.
9196	*
9197	* @param pVCpu The cross context virtual CPU structure.
9198	* @param pVmcsInfo The VMCS info. object.
9199	*
9200	* @remarks This function assumes our cache of the VMCS controls
9201	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9202	*/
9203	static uint32_t hmR0VmxCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9204	{
9205	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9206	#define HMVMX_CHECK_BREAK(expr, err) do { \
9207	if (!(expr)) { uError = (err); break; } \
9208	} while (0)
9209
9210	int rc;
9211	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9212	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9213	uint32_t uError = VMX_IGS_ERROR;
9214	uint32_t u32Val;
9215	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9216
9217	do
9218	{
9219	/*
9220	* CR0.
9221	*/
9222	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9223	uint64_t fSetCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9224	uint64_t const fZapCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9225	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
9226	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9227	if (fUnrestrictedGuest)
9228	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
9229
9230	uint64_t u64GuestCr0;
9231	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64GuestCr0);
9232	AssertRC(rc);
9233	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9234	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9235	if ( !fUnrestrictedGuest
9236	&& (u64GuestCr0 & X86_CR0_PG)
9237	&& !(u64GuestCr0 & X86_CR0_PE))
9238	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9239
9240	/*
9241	* CR4.
9242	*/
9243	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9244	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9245	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9246
9247	uint64_t u64GuestCr4;
9248	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64GuestCr4);
9249	AssertRC(rc);
9250	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9251	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9252
9253	/*
9254	* IA32_DEBUGCTL MSR.
9255	*/
9256	uint64_t u64Val;
9257	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9258	AssertRC(rc);
9259	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9260	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9261	{
9262	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9263	}
9264	uint64_t u64DebugCtlMsr = u64Val;
9265
9266	#ifdef VBOX_STRICT
9267	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9268	AssertRC(rc);
9269	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9270	#endif
9271	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9272
9273	/*
9274	* RIP and RFLAGS.
9275	*/
9276	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
9277	AssertRC(rc);
9278	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9279	if ( !fLongModeGuest
9280	\|\| !pCtx->cs.Attr.n.u1Long)
9281	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9282	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9283	* must be identical if the "IA-32e mode guest" VM-entry
9284	* control is 1 and CS.L is 1. No check applies if the
9285	* CPU supports 64 linear-address bits. */
9286
9287	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9288	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9289	AssertRC(rc);
9290	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9291	VMX_IGS_RFLAGS_RESERVED);
9292	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9293	uint32_t const u32Eflags = u64Val;
9294
9295	if ( fLongModeGuest
9296	\|\| ( fUnrestrictedGuest
9297	&& !(u64GuestCr0 & X86_CR0_PE)))
9298	{
9299	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9300	}
9301
9302	uint32_t u32EntryInfo;
9303	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9304	AssertRC(rc);
9305	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9306	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9307
9308	/*
9309	* 64-bit checks.
9310	*/
9311	if (fLongModeGuest)
9312	{
9313	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9314	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9315	}
9316
9317	if ( !fLongModeGuest
9318	&& (u64GuestCr4 & X86_CR4_PCIDE))
9319	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9320
9321	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9322	* 51:32 beyond the processor's physical-address width are 0. */
9323
9324	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9325	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9326	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9327
9328	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9329	AssertRC(rc);
9330	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9331
9332	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9333	AssertRC(rc);
9334	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9335
9336	/*
9337	* PERF_GLOBAL MSR.
9338	*/
9339	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9340	{
9341	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9342	AssertRC(rc);
9343	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9344	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9345	}
9346
9347	/*
9348	* PAT MSR.
9349	*/
9350	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9351	{
9352	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9353	AssertRC(rc);
9354	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9355	for (unsigned i = 0; i < 8; i++)
9356	{
9357	uint8_t u8Val = (u64Val & 0xff);
9358	if ( u8Val != 0 /* UC */
9359	&& u8Val != 1 /* WC */
9360	&& u8Val != 4 /* WT */
9361	&& u8Val != 5 /* WP */
9362	&& u8Val != 6 /* WB */
9363	&& u8Val != 7 /* UC- */)
9364	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9365	u64Val >>= 8;
9366	}
9367	}
9368
9369	/*
9370	* EFER MSR.
9371	*/
9372	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9373	{
9374	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9375	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9376	AssertRC(rc);
9377	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9378	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9379	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9380	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9381	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9382	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9383	* iemVmxVmentryCheckGuestState(). */
9384	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9385	\|\| !(u64GuestCr0 & X86_CR0_PG)
9386	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9387	VMX_IGS_EFER_LMA_LME_MISMATCH);
9388	}
9389
9390	/*
9391	* Segment registers.
9392	*/
9393	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9394	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9395	if (!(u32Eflags & X86_EFL_VM))
9396	{
9397	/* CS */
9398	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9399	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9400	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9401	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9402	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9403	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9404	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9405	/* CS cannot be loaded with NULL in protected mode. */
9406	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9407	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9408	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9409	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9410	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9411	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9412	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9413	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9414	else
9415	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9416
9417	/* SS */
9418	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9419	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9420	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9421	if ( !(pCtx->cr0 & X86_CR0_PE)
9422	\|\| pCtx->cs.Attr.n.u4Type == 3)
9423	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9424
9425	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9426	{
9427	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9428	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9429	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9430	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9431	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9432	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9433	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9434	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9435	}
9436
9437	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9438	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9439	{
9440	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9441	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9442	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9443	\|\| pCtx->ds.Attr.n.u4Type > 11
9444	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9445	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9446	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9447	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9448	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9449	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9450	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9451	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9452	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9453	}
9454	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9455	{
9456	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9457	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9458	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9459	\|\| pCtx->es.Attr.n.u4Type > 11
9460	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9461	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9462	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9463	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9464	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9465	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9466	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9467	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9468	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9469	}
9470	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9471	{
9472	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9473	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9474	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9475	\|\| pCtx->fs.Attr.n.u4Type > 11
9476	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9477	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9478	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9479	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9480	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9481	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9482	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9483	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9484	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9485	}
9486	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9487	{
9488	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9489	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9490	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9491	\|\| pCtx->gs.Attr.n.u4Type > 11
9492	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9493	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9494	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9495	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9496	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9497	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9498	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9499	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9500	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9501	}
9502	/* 64-bit capable CPUs. */
9503	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9504	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9505	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9506	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9507	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9508	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9509	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9510	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9511	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9512	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9513	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9514	}
9515	else
9516	{
9517	/* V86 mode checks. */
9518	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9519	if (pVmcsInfo->RealMode.fRealOnV86Active)
9520	{
9521	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9522	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9523	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9524	}
9525	else
9526	{
9527	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9528	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9529	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9530	}
9531
9532	/* CS */
9533	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9534	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9535	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9536	/* SS */
9537	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9538	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9539	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9540	/* DS */
9541	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9542	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9543	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9544	/* ES */
9545	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9546	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9547	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9548	/* FS */
9549	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9550	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9551	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9552	/* GS */
9553	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9554	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9555	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9556	/* 64-bit capable CPUs. */
9557	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9558	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9559	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9560	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9561	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9562	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9563	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9564	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9565	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9566	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9567	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9568	}
9569
9570	/*
9571	* TR.
9572	*/
9573	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9574	/* 64-bit capable CPUs. */
9575	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9576	if (fLongModeGuest)
9577	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9578	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9579	else
9580	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9581	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9582	VMX_IGS_TR_ATTR_TYPE_INVALID);
9583	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9584	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9585	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9586	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9587	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9588	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9589	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9590	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9591
9592	/*
9593	* GDTR and IDTR (64-bit capable checks).
9594	*/
9595	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9596	AssertRC(rc);
9597	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9598
9599	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9600	AssertRC(rc);
9601	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9602
9603	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9604	AssertRC(rc);
9605	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9606
9607	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9608	AssertRC(rc);
9609	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9610
9611	/*
9612	* Guest Non-Register State.
9613	*/
9614	/* Activity State. */
9615	uint32_t u32ActivityState;
9616	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9617	AssertRC(rc);
9618	HMVMX_CHECK_BREAK( !u32ActivityState
9619	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9620	VMX_IGS_ACTIVITY_STATE_INVALID);
9621	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9622	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9623	uint32_t u32IntrState;
9624	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9625	AssertRC(rc);
9626	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9627	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9628	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9629
9630	/** @todo Activity state and injecting interrupts. Left as a todo since we
9631	* currently don't use activity states but ACTIVE. */
9632
9633	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9634	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9635
9636	/* Guest interruptibility-state. */
9637	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9638	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9639	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9640	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9641	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9642	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9643	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9644	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9645	{
9646	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9647	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9648	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9649	}
9650	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9651	{
9652	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9653	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9654	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9655	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9656	}
9657	/** @todo Assumes the processor is not in SMM. */
9658	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9659	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9660	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9661	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9662	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9663	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9664	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9665	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9666
9667	/* Pending debug exceptions. */
9668	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9669	AssertRC(rc);
9670	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9671	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9672	u32Val = u64Val; /* For pending debug exceptions checks below. */
9673
9674	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9675	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9676	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9677	{
9678	if ( (u32Eflags & X86_EFL_TF)
9679	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9680	{
9681	/* Bit 14 is PendingDebug.BS. */
9682	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9683	}
9684	if ( !(u32Eflags & X86_EFL_TF)
9685	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9686	{
9687	/* Bit 14 is PendingDebug.BS. */
9688	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9689	}
9690	}
9691
9692	/* VMCS link pointer. */
9693	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9694	AssertRC(rc);
9695	if (u64Val != UINT64_C(0xffffffffffffffff))
9696	{
9697	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9698	/** @todo Bits beyond the processor's physical-address width MBZ. */
9699	/** @todo SMM checks. */
9700	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
9701	Assert(pVmcsInfo->pvShadowVmcs);
9702	VMXVMCSREVID VmcsRevId;
9703	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
9704	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
9705	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
9706	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
9707	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
9708	}
9709
9710	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9711	* not using nested paging? */
9712	if ( pVM->hm.s.fNestedPaging
9713	&& !fLongModeGuest
9714	&& CPUMIsGuestInPAEModeEx(pCtx))
9715	{
9716	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9717	AssertRC(rc);
9718	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9719
9720	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9721	AssertRC(rc);
9722	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9723
9724	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9725	AssertRC(rc);
9726	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9727
9728	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9729	AssertRC(rc);
9730	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9731	}
9732
9733	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9734	if (uError == VMX_IGS_ERROR)
9735	uError = VMX_IGS_REASON_NOT_FOUND;
9736	} while (0);
9737
9738	pVCpu->hm.s.u32HMError = uError;
9739	return uError;
9740
9741	#undef HMVMX_ERROR_BREAK
9742	#undef HMVMX_CHECK_BREAK
9743	}
9744
9745
9746	/**
9747	* Map the APIC-access page for virtualizing APIC accesses.
9748	*
9749	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
9750	* this not done as part of exporting guest state, see @bugref{8721}.
9751	*
9752	* @returns VBox status code.
9753	* @param pVCpu The cross context virtual CPU structure.
9754	*/
9755	static int hmR0VmxMapHCApicAccessPage(PVMCPUCC pVCpu)
9756	{
9757	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9758	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9759
9760	Assert(PDMHasApic(pVM));
9761	Assert(u64MsrApicBase);
9762
9763	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9764	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9765
9766	/* Unalias the existing mapping. */
9767	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9768	AssertRCReturn(rc, rc);
9769
9770	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9771	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9772	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9773	AssertRCReturn(rc, rc);
9774
9775	/* Update the per-VCPU cache of the APIC base MSR. */
9776	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9777	return VINF_SUCCESS;
9778	}
9779
9780
9781	/**
9782	* Worker function passed to RTMpOnSpecific() that is to be called on the target
9783	* CPU.
9784	*
9785	* @param idCpu The ID for the CPU the function is called on.
9786	* @param pvUser1 Null, not used.
9787	* @param pvUser2 Null, not used.
9788	*/
9789	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
9790	{
9791	RT_NOREF3(idCpu, pvUser1, pvUser2);
9792	VMXDispatchHostNmi();
9793	}
9794
9795
9796	/**
9797	* Dispatching an NMI on the host CPU that received it.
9798	*
9799	* @returns VBox status code.
9800	* @param pVCpu The cross context virtual CPU structure.
9801	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
9802	* executing when receiving the host NMI in VMX non-root
9803	* operation.
9804	*/
9805	static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9806	{
9807	RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
9808	Assert(idCpu != NIL_RTCPUID);
9809
9810	/*
9811	* We don't want to delay dispatching the NMI any more than we have to. However,
9812	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
9813	* after executing guest or nested-guest code for the following reasons:
9814	*
9815	* - We would need to perform VMREADs with interrupts disabled and is orders of
9816	* magnitude worse when we run as a nested hypervisor without VMCS shadowing
9817	* supported by the host hypervisor.
9818	*
9819	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
9820	* longer period of time just for handling an edge case like host NMIs which do
9821	* not occur nearly as frequently as other VM-exits.
9822	*
9823	* Let's cover the most likely scenario first. Check if we are on the target CPU
9824	* and dispatch the NMI right away. This should be much faster than calling into
9825	* RTMpOnSpecific() machinery.
9826	*/
9827	bool fDispatched = false;
9828	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
9829	if (idCpu == RTMpCpuId())
9830	{
9831	VMXDispatchHostNmi();
9832	fDispatched = true;
9833	}
9834	ASMSetFlags(fEFlags);
9835	if (fDispatched)
9836	{
9837	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9838	return VINF_SUCCESS;
9839	}
9840
9841	/*
9842	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
9843	* there should be no race or recursion even if we are unlucky enough to be preempted
9844	* (to the target CPU) without dispatching the host NMI above.
9845	*/
9846	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
9847	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
9848	}
9849
9850
9851	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9852	/**
9853	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9854	* nested-guest using hardware-assisted VMX.
9855	*
9856	* @param pVCpu The cross context virtual CPU structure.
9857	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9858	* @param pVmcsInfoGst The guest VMCS info. object.
9859	*/
9860	static void hmR0VmxMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9861	{
9862	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
9863	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9864	Assert(pu64MsrBitmap);
9865
9866	/*
9867	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9868	* MSR that is intercepted by the guest is also intercepted while executing the
9869	* nested-guest using hardware-assisted VMX.
9870	*
9871	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
9872	* nested-guest VM-exit even if the outer guest is not intercepting some
9873	* MSRs. We cannot assume the caller has initialized the nested-guest
9874	* MSR bitmap in this case.
9875	*
9876	* The nested hypervisor may also switch whether it uses MSR bitmaps for
9877	* each VM-entry, hence initializing it once per-VM while setting up the
9878	* nested-guest VMCS is not sufficient.
9879	*/
9880	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9881	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
9882	{
9883	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9884	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9885	Assert(pu64MsrBitmapNstGst);
9886	Assert(pu64MsrBitmapGst);
9887
9888	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
9889	for (uint32_t i = 0; i < cFrags; i++)
9890	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9891	}
9892	else
9893	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
9894	}
9895
9896
9897	/**
9898	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9899	* hardware-assisted VMX execution of the nested-guest.
9900	*
9901	* For a guest, we don't modify these controls once we set up the VMCS and hence
9902	* this function is never called.
9903	*
9904	* For nested-guests since the nested hypervisor provides these controls on every
9905	* nested-guest VM-entry and could potentially change them everytime we need to
9906	* merge them before every nested-guest VM-entry.
9907	*
9908	* @returns VBox status code.
9909	* @param pVCpu The cross context virtual CPU structure.
9910	*/
9911	static int hmR0VmxMergeVmcsNested(PVMCPUCC pVCpu)
9912	{
9913	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9914	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9915	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9916	Assert(pVmcsNstGst);
9917
9918	/*
9919	* Merge the controls with the requirements of the guest VMCS.
9920	*
9921	* We do not need to validate the nested-guest VMX features specified in the nested-guest
9922	* VMCS with the features supported by the physical CPU as it's already done by the
9923	* VMLAUNCH/VMRESUME instruction emulation.
9924	*
9925	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
9926	* derived from the VMX features supported by the physical CPU.
9927	*/
9928
9929	/* Pin-based VM-execution controls. */
9930	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9931
9932	/* Processor-based VM-execution controls. */
9933	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9934	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9935	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9936	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9937	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9938
9939	/* Secondary processor-based VM-execution controls. */
9940	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9941	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9942	\| VMX_PROC_CTLS2_INVPCID
9943	\| VMX_PROC_CTLS2_VMCS_SHADOWING
9944	\| VMX_PROC_CTLS2_RDTSCP
9945	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9946	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9947	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9948	\| VMX_PROC_CTLS2_VMFUNC));
9949
9950	/*
9951	* VM-entry controls:
9952	* These controls contains state that depends on the nested-guest state (primarily
9953	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
9954	* VM-exit. Although the nested hypervisor cannot change it, we need to in order to
9955	* properly continue executing the nested-guest if the EFER MSR changes but does not
9956	* cause a nested-guest VM-exits.
9957	*
9958	* VM-exit controls:
9959	* These controls specify the host state on return. We cannot use the controls from
9960	* the nested hypervisor state as is as it would contain the guest state rather than
9961	* the host state. Since the host state is subject to change (e.g. preemption, trips
9962	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9963	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9964	*
9965	* VM-entry MSR-load:
9966	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
9967	* context by the VMLAUNCH/VMRESUME instruction emulation.
9968	*
9969	* VM-exit MSR-store:
9970	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
9971	* back into the VM-exit MSR-store area.
9972	*
9973	* VM-exit MSR-load areas:
9974	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
9975	* can entirely ignore what the nested hypervisor wants to load here.
9976	*/
9977
9978	/*
9979	* Exception bitmap.
9980	*
9981	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
9982	* here (and avoid doing anything while exporting nested-guest state), but to keep the
9983	* code more flexible if intercepting exceptions become more dynamic in the future we do
9984	* it as part of exporting the nested-guest state.
9985	*/
9986	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9987
9988	/*
9989	* CR0/CR4 guest/host mask.
9990	*
9991	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
9992	* cause VM-exits, so we need to merge them here.
9993	*/
9994	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9995	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9996
9997	/*
9998	* Page-fault error-code mask and match.
9999	*
10000	* Although we require unrestricted guest execution (and thereby nested-paging) for
10001	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10002	* normally intercept #PFs, it might intercept them for debugging purposes.
10003	*
10004	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10005	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10006	*/
10007	uint32_t u32XcptPFMask;
10008	uint32_t u32XcptPFMatch;
10009	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10010	{
10011	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10012	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10013	}
10014	else
10015	{
10016	u32XcptPFMask = 0;
10017	u32XcptPFMatch = 0;
10018	}
10019
10020	/*
10021	* Pause-Loop exiting.
10022	*/
10023	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10024	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10025
10026	/*
10027	* Pending debug exceptions.
10028	* Currently just copy whatever the nested-guest provides us.
10029	*/
10030	uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
10031
10032	/*
10033	* I/O Bitmap.
10034	*
10035	* We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
10036	* intercept all I/O port accesses.
10037	*/
10038	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10039	Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
10040
10041	/*
10042	* VMCS shadowing.
10043	*
10044	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10045	* enabled while executing the nested-guest.
10046	*/
10047	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10048
10049	/*
10050	* APIC-access page.
10051	*/
10052	RTHCPHYS HCPhysApicAccess;
10053	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10054	{
10055	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10056	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10057
10058	/** @todo NSTVMX: This is not really correct but currently is required to make
10059	* things work. We need to re-enable the page handler when we fallback to
10060	* IEM execution of the nested-guest! */
10061	PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
10062
10063	void *pvPage;
10064	PGMPAGEMAPLOCK PgLockApicAccess;
10065	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
10066	if (RT_SUCCESS(rc))
10067	{
10068	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10069	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10070
10071	/** @todo Handle proper releasing of page-mapping lock later. */
10072	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
10073	}
10074	else
10075	return rc;
10076	}
10077	else
10078	HCPhysApicAccess = 0;
10079
10080	/*
10081	* Virtual-APIC page and TPR threshold.
10082	*/
10083	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10084	RTHCPHYS HCPhysVirtApic;
10085	uint32_t u32TprThreshold;
10086	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10087	{
10088	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10089	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10090
10091	void *pvPage;
10092	PGMPAGEMAPLOCK PgLockVirtApic;
10093	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
10094	if (RT_SUCCESS(rc))
10095	{
10096	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10097	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10098
10099	/** @todo Handle proper releasing of page-mapping lock later. */
10100	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
10101	}
10102	else
10103	return rc;
10104
10105	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10106	}
10107	else
10108	{
10109	HCPhysVirtApic = 0;
10110	u32TprThreshold = 0;
10111
10112	/*
10113	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10114	* used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
10115	* be taken care of by EPT/shadow paging.
10116	*/
10117	if (pVM->hm.s.fAllow64BitGuests)
10118	{
10119	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10120	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10121	}
10122	}
10123
10124	/*
10125	* Validate basic assumptions.
10126	*/
10127	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10128	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10129	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10130
10131	/*
10132	* Commit it to the nested-guest VMCS.
10133	*/
10134	int rc = VINF_SUCCESS;
10135	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10136	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10137	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10138	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10139	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10140	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10141	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10142	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10143	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10144	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10145	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10146	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10147	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10148	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10149	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10150	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10151	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10152	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10153	{
10154	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10155	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10156	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10157	}
10158	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10159	{
10160	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10161	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10162	}
10163	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10164	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10165	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
10166	AssertRC(rc);
10167
10168	/*
10169	* Update the nested-guest VMCS cache.
10170	*/
10171	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10172	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10173	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10174	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10175	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10176	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10177	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10178	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10179	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10180
10181	/*
10182	* We need to flush the TLB if we are switching the APIC-access page address.
10183	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10184	*/
10185	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10186	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10187
10188	/*
10189	* MSR bitmap.
10190	*
10191	* The MSR bitmap address has already been initialized while setting up the nested-guest
10192	* VMCS, here we need to merge the MSR bitmaps.
10193	*/
10194	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10195	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10196
10197	return VINF_SUCCESS;
10198	}
10199	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10200
10201
10202	/**
10203	* Does the preparations before executing guest code in VT-x.
10204	*
10205	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10206	* recompiler/IEM. We must be cautious what we do here regarding committing
10207	* guest-state information into the VMCS assuming we assuredly execute the
10208	* guest in VT-x mode.
10209	*
10210	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10211	* the common-state (TRPM/forceflags), we must undo those changes so that the
10212	* recompiler/IEM can (and should) use them when it resumes guest execution.
10213	* Otherwise such operations must be done when we can no longer exit to ring-3.
10214	*
10215	* @returns Strict VBox status code (i.e. informational status codes too).
10216	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10217	* have been disabled.
10218	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10219	* pending events).
10220	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10221	* double-fault into the guest.
10222	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10223	* dispatched directly.
10224	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10225	*
10226	* @param pVCpu The cross context virtual CPU structure.
10227	* @param pVmxTransient The VMX-transient structure.
10228	* @param fStepping Whether we are single-stepping the guest in the
10229	* hypervisor debugger. Makes us ignore some of the reasons
10230	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10231	* if event dispatching took place.
10232	*/
10233	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10234	{
10235	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10236
10237	Log4Func(("fIsNested=%RTbool fStepping=%RTbool\n", pVmxTransient->fIsNestedGuest, fStepping));
10238
10239	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10240	if (pVmxTransient->fIsNestedGuest)
10241	{
10242	RT_NOREF2(pVCpu, fStepping);
10243	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10244	return VINF_EM_RESCHEDULE_REM;
10245	}
10246	#endif
10247
10248	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10249	PGMRZDynMapFlushAutoSet(pVCpu);
10250	#endif
10251
10252	/*
10253	* Check and process force flag actions, some of which might require us to go back to ring-3.
10254	*/
10255	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10256	if (rcStrict == VINF_SUCCESS)
10257	{
10258	/* FFs don't get set all the time. */
10259	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10260	if ( pVmxTransient->fIsNestedGuest
10261	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10262	{
10263	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10264	return VINF_VMX_VMEXIT;
10265	}
10266	#endif
10267	}
10268	else
10269	return rcStrict;
10270
10271	/*
10272	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10273	*/
10274	/** @todo Doing this from ring-3 after VM setup phase causes a
10275	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
10276	* idea why atm. */
10277	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10278	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10279	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10280	&& PDMHasApic(pVM))
10281	{
10282	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10283	AssertRCReturn(rc, rc);
10284	}
10285
10286	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10287	/*
10288	* Merge guest VMCS controls with the nested-guest VMCS controls.
10289	*
10290	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10291	* saved state), we should be okay with merging controls as we initialize the
10292	* guest VMCS controls as part of VM setup phase.
10293	*/
10294	if ( pVmxTransient->fIsNestedGuest
10295	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10296	{
10297	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10298	AssertRCReturn(rc, rc);
10299	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10300	}
10301	#endif
10302
10303	/*
10304	* Evaluate events to be injected into the guest.
10305	*
10306	* Events in TRPM can be injected without inspecting the guest state.
10307	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10308	* guest to cause a VM-exit the next time they are ready to receive the event.
10309	*
10310	* With nested-guests, evaluating pending events may cause VM-exits. Also, verify
10311	* that the event in TRPM that we will inject using hardware-assisted VMX is -not-
10312	* subject to interecption. Otherwise, we should have checked and injected them
10313	* manually elsewhere (IEM).
10314	*/
10315	if (TRPMHasTrap(pVCpu))
10316	{
10317	Assert(!pVmxTransient->fIsNestedGuest \|\| !CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
10318	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10319	}
10320
10321	uint32_t fIntrState;
10322	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10323
10324	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10325	/*
10326	* While evaluating pending events if something failed (unlikely) or if we were
10327	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10328	*/
10329	if (rcStrict != VINF_SUCCESS)
10330	return rcStrict;
10331	if ( pVmxTransient->fIsNestedGuest
10332	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10333	{
10334	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10335	return VINF_VMX_VMEXIT;
10336	}
10337	#else
10338	Assert(rcStrict == VINF_SUCCESS);
10339	#endif
10340
10341	/*
10342	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10343	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10344	* also result in triple-faulting the VM.
10345	*
10346	* With nested-guests, the above does not apply since unrestricted guest execution is a
10347	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10348	*/
10349	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10350	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10351	{ /* likely */ }
10352	else
10353	{
10354	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10355	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10356	return rcStrict;
10357	}
10358
10359	/*
10360	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10361	* import CR3 themselves. We will need to update them here, as even as late as the above
10362	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10363	* the below force flags to be set.
10364	*/
10365	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10366	{
10367	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10368	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10369	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10370	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10371	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10372	}
10373	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10374	{
10375	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10376	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10377	}
10378
10379	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10380	/* Paranoia. */
10381	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10382	#endif
10383
10384	/*
10385	* No longjmps to ring-3 from this point on!!!
10386	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10387	* This also disables flushing of the R0-logger instance (if any).
10388	*/
10389	VMMRZCallRing3Disable(pVCpu);
10390
10391	/*
10392	* Export the guest state bits.
10393	*
10394	* We cannot perform longjmps while loading the guest state because we do not preserve the
10395	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10396	* CPU migration.
10397	*
10398	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10399	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10400	*/
10401	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10402	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10403	{ /* likely */ }
10404	else
10405	{
10406	VMMRZCallRing3Enable(pVCpu);
10407	return rcStrict;
10408	}
10409
10410	/*
10411	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10412	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10413	* preemption disabled for a while. Since this is purely to aid the
10414	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10415	* disable interrupt on NT.
10416	*
10417	* We need to check for force-flags that could've possible been altered since we last
10418	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10419	* see @bugref{6398}).
10420	*
10421	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10422	* to ring-3 before executing guest code.
10423	*/
10424	pVmxTransient->fEFlags = ASMIntDisableFlags();
10425
10426	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10427	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10428	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10429	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10430	{
10431	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10432	{
10433	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10434	/*
10435	* If we are executing a nested-guest make sure that we should intercept subsequent
10436	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10437	* the VM-exit instruction emulation happy.
10438	*/
10439	if (pVmxTransient->fIsNestedGuest)
10440	CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
10441	#endif
10442
10443	/*
10444	* We've injected any pending events. This is really the point of no return (to ring-3).
10445	*
10446	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10447	* returns from this function, so do -not- enable them here.
10448	*/
10449	pVCpu->hm.s.Event.fPending = false;
10450	return VINF_SUCCESS;
10451	}
10452
10453	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10454	rcStrict = VINF_EM_RAW_INTERRUPT;
10455	}
10456	else
10457	{
10458	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10459	rcStrict = VINF_EM_RAW_TO_R3;
10460	}
10461
10462	ASMSetFlags(pVmxTransient->fEFlags);
10463	VMMRZCallRing3Enable(pVCpu);
10464
10465	return rcStrict;
10466	}
10467
10468
10469	/**
10470	* Final preparations before executing guest code using hardware-assisted VMX.
10471	*
10472	* We can no longer get preempted to a different host CPU and there are no returns
10473	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10474	* failures), this function is not intended to fail sans unrecoverable hardware
10475	* errors.
10476	*
10477	* @param pVCpu The cross context virtual CPU structure.
10478	* @param pVmxTransient The VMX-transient structure.
10479	*
10480	* @remarks Called with preemption disabled.
10481	* @remarks No-long-jump zone!!!
10482	*/
10483	static void hmR0VmxPreRunGuestCommitted(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10484	{
10485	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10486	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10487	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10488	Assert(!pVCpu->hm.s.Event.fPending);
10489
10490	/*
10491	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10492	*/
10493	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10494	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10495
10496	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10497	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10498	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10499	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10500
10501	if (!CPUMIsGuestFPUStateActive(pVCpu))
10502	{
10503	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10504	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10505	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10506	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10507	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10508	}
10509
10510	/*
10511	* Re-export the host state bits as we may've been preempted (only happens when
10512	* thread-context hooks are used or when the VM start function changes) or if
10513	* the host CR0 is modified while loading the guest FPU state above.
10514	*
10515	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10516	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10517	* see @bugref{8432}.
10518	*
10519	* This may also happen when switching to/from a nested-guest VMCS without leaving
10520	* ring-0.
10521	*/
10522	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10523	{
10524	hmR0VmxExportHostState(pVCpu);
10525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10526	}
10527	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10528
10529	/*
10530	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10531	*/
10532	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10533	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10534	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10535
10536	/*
10537	* Store status of the shared guest/host debug state at the time of VM-entry.
10538	*/
10539	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10540	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10541
10542	/*
10543	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10544	* more than one conditional check. The post-run side of our code shall determine
10545	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10546	*/
10547	if (pVmcsInfo->pbVirtApic)
10548	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10549
10550	/*
10551	* Update the host MSRs values in the VM-exit MSR-load area.
10552	*/
10553	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10554	{
10555	if (pVmcsInfo->cExitMsrLoad > 0)
10556	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10557	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10558	}
10559
10560	/*
10561	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10562	* VMX-preemption timer based on the next virtual sync clock deadline.
10563	*/
10564	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10565	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10566	{
10567	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10568	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10569	}
10570
10571	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10572	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10573	if (!fIsRdtscIntercepted)
10574	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10575	else
10576	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10577
10578	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10579	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10580	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10581	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Record the error reporting info. with the current host CPU. */
10582	pVmcsInfo->idHostCpuState = idCurrentCpu; /* Record the CPU for which the host-state has been exported. */
10583	pVmcsInfo->idHostCpuExec = idCurrentCpu; /* Record the CPU on which we shall execute. */
10584
10585	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10586
10587	TMNotifyStartOfExecution(pVM, pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10588	as we're about to start executing the guest. */
10589
10590	/*
10591	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10592	*
10593	* This is done this late as updating the TSC offsetting/preemption timer above
10594	* figures out if we can skip intercepting RDTSCP by calculating the number of
10595	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10596	*/
10597	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10598	&& !fIsRdtscIntercepted)
10599	{
10600	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10601
10602	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10603	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10604	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10605	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10606	AssertRC(rc);
10607	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10608	pVmxTransient->fRemoveTscAuxMsr = true;
10609	}
10610
10611	#ifdef VBOX_STRICT
10612	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10613	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10614	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10615	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10616	#endif
10617
10618	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10619	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10620	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10621	* see @bugref{9180#c54}. */
10622	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10623	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10624	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10625	#endif
10626	}
10627
10628
10629	/**
10630	* First C routine invoked after running guest code using hardware-assisted VMX.
10631	*
10632	* @param pVCpu The cross context virtual CPU structure.
10633	* @param pVmxTransient The VMX-transient structure.
10634	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10635	*
10636	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10637	*
10638	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10639	* unconditionally when it is safe to do so.
10640	*/
10641	static void hmR0VmxPostRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10642	{
10643	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10644
10645	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10646	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10647	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10648	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10649	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10650	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10651
10652	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10653	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10654	{
10655	uint64_t uGstTsc;
10656	if (!pVmxTransient->fIsNestedGuest)
10657	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10658	else
10659	{
10660	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10661	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10662	}
10663	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10664	}
10665
10666	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10667	TMNotifyEndOfExecution(pVCpu->CTX_SUFF(pVM), pVCpu); /* Notify TM that the guest is no longer running. */
10668	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10669
10670	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10671	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10672	#ifdef VBOX_STRICT
10673	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10674	#endif
10675	Assert(!ASMIntAreEnabled());
10676	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10677	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10678
10679	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
10680	/*
10681	* Clean all the VMCS fields in the transient structure before reading
10682	* anything from the VMCS.
10683	*/
10684	pVmxTransient->uExitReason = 0;
10685	pVmxTransient->uExitIntErrorCode = 0;
10686	pVmxTransient->uExitQual = 0;
10687	pVmxTransient->uGuestLinearAddr = 0;
10688	pVmxTransient->uExitIntInfo = 0;
10689	pVmxTransient->cbExitInstr = 0;
10690	pVmxTransient->ExitInstrInfo.u = 0;
10691	pVmxTransient->uEntryIntInfo = 0;
10692	pVmxTransient->uEntryXcptErrorCode = 0;
10693	pVmxTransient->cbEntryInstr = 0;
10694	pVmxTransient->uIdtVectoringInfo = 0;
10695	pVmxTransient->uIdtVectoringErrorCode = 0;
10696	#endif
10697
10698	/*
10699	* Save the basic VM-exit reason and check if the VM-entry failed.
10700	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10701	*/
10702	uint32_t uExitReason;
10703	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10704	AssertRC(rc);
10705	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10706	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10707
10708	/*
10709	* Log the VM-exit before logging anything else as otherwise it might be a
10710	* tad confusing what happens before and after the world-switch.
10711	*/
10712	HMVMX_LOG_EXIT(pVCpu, uExitReason);
10713
10714	/*
10715	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
10716	* bitmap permissions, if it was added before VM-entry.
10717	*/
10718	if (pVmxTransient->fRemoveTscAuxMsr)
10719	{
10720	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10721	pVmxTransient->fRemoveTscAuxMsr = false;
10722	}
10723
10724	/*
10725	* Check if VMLAUNCH/VMRESUME succeeded.
10726	* If this failed, we cause a guru meditation and cease further execution.
10727	*
10728	* However, if we are executing a nested-guest we might fail if we use the
10729	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10730	*/
10731	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10732	{
10733	/*
10734	* Update the VM-exit history array here even if the VM-entry failed due to:
10735	* - Invalid guest state.
10736	* - MSR loading.
10737	* - Machine-check event.
10738	*
10739	* In any of the above cases we will still have a "valid" VM-exit reason
10740	* despite @a fVMEntryFailed being false.
10741	*
10742	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10743	*
10744	* Note! We don't have CS or RIP at this point. Will probably address that later
10745	* by amending the history entry added here.
10746	*/
10747	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10748	UINT64_MAX, uHostTsc);
10749
10750	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10751	{
10752	VMMRZCallRing3Enable(pVCpu);
10753
10754	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10755	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10756
10757	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
10758	hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
10759	#endif
10760	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10761	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10762	AssertRC(rc);
10763	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10764	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10765	AssertRC(rc);
10766	#else
10767	/*
10768	* Import the guest-interruptibility state always as we need it while evaluating
10769	* injecting events on re-entry.
10770	*
10771	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10772	* checking for real-mode while exporting the state because all bits that cause
10773	* mode changes wrt CR0 are intercepted.
10774	*/
10775	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10776	AssertRC(rc);
10777	#endif
10778
10779	/*
10780	* Sync the TPR shadow with our APIC state.
10781	*/
10782	if ( !pVmxTransient->fIsNestedGuest
10783	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10784	{
10785	Assert(pVmcsInfo->pbVirtApic);
10786	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10787	{
10788	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10789	AssertRC(rc);
10790	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10791	}
10792	}
10793
10794	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10795	return;
10796	}
10797	}
10798	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10799	else if (pVmxTransient->fIsNestedGuest)
10800	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10801	#endif
10802	else
10803	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10804
10805	VMMRZCallRing3Enable(pVCpu);
10806	}
10807
10808
10809	/**
10810	* Runs the guest code using hardware-assisted VMX the normal way.
10811	*
10812	* @returns VBox status code.
10813	* @param pVCpu The cross context virtual CPU structure.
10814	* @param pcLoops Pointer to the number of executed loops.
10815	*/
10816	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
10817	{
10818	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10819	Assert(pcLoops);
10820	Assert(*pcLoops <= cMaxResumeLoops);
10821	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10822
10823	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10824	/*
10825	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
10826	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
10827	* guest VMCS while entering the VMX ring-0 session.
10828	*/
10829	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10830	{
10831	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
10832	if (RT_SUCCESS(rc))
10833	{ /* likely */ }
10834	else
10835	{
10836	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
10837	return rc;
10838	}
10839	}
10840	#endif
10841
10842	VMXTRANSIENT VmxTransient;
10843	RT_ZERO(VmxTransient);
10844	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10845
10846	/* Paranoia. */
10847	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10848
10849	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10850	for (;;)
10851	{
10852	Assert(!HMR0SuspendPending());
10853	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10854	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10855
10856	/*
10857	* Preparatory work for running nested-guest code, this may force us to
10858	* return to ring-3.
10859	*
10860	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10861	*/
10862	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10863	if (rcStrict != VINF_SUCCESS)
10864	break;
10865
10866	/* Interrupts are disabled at this point! */
10867	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10868	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10869	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10870	/* Interrupts are re-enabled at this point! */
10871
10872	/*
10873	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10874	*/
10875	if (RT_SUCCESS(rcRun))
10876	{ /* very likely */ }
10877	else
10878	{
10879	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10880	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10881	return rcRun;
10882	}
10883
10884	/*
10885	* Profile the VM-exit.
10886	*/
10887	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10888	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10889	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10890	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10891	HMVMX_START_EXIT_DISPATCH_PROF();
10892
10893	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10894
10895	/*
10896	* Handle the VM-exit.
10897	*/
10898	#ifdef HMVMX_USE_FUNCTION_TABLE
10899	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10900	#else
10901	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10902	#endif
10903	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10904	if (rcStrict == VINF_SUCCESS)
10905	{
10906	if (++(*pcLoops) <= cMaxResumeLoops)
10907	continue;
10908	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10909	rcStrict = VINF_EM_RAW_INTERRUPT;
10910	}
10911	break;
10912	}
10913
10914	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10915	return rcStrict;
10916	}
10917
10918
10919	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10920	/**
10921	* Runs the nested-guest code using hardware-assisted VMX.
10922	*
10923	* @returns VBox status code.
10924	* @param pVCpu The cross context virtual CPU structure.
10925	* @param pcLoops Pointer to the number of executed loops.
10926	*
10927	* @sa hmR0VmxRunGuestCodeNormal.
10928	*/
10929	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
10930	{
10931	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10932	Assert(pcLoops);
10933	Assert(*pcLoops <= cMaxResumeLoops);
10934	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10935
10936	/*
10937	* Switch to the nested-guest VMCS as we may have transitioned from executing the
10938	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
10939	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
10940	*/
10941	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10942	{
10943	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
10944	if (RT_SUCCESS(rc))
10945	{ /* likely */ }
10946	else
10947	{
10948	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
10949	return rc;
10950	}
10951	}
10952
10953	VMXTRANSIENT VmxTransient;
10954	RT_ZERO(VmxTransient);
10955	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10956	VmxTransient.fIsNestedGuest = true;
10957
10958	/* Paranoia. */
10959	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10960
10961	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10962	for (;;)
10963	{
10964	Assert(!HMR0SuspendPending());
10965	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10966	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10967
10968	/*
10969	* Preparatory work for running guest code, this may force us to
10970	* return to ring-3.
10971	*
10972	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10973	*/
10974	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10975	if (rcStrict != VINF_SUCCESS)
10976	break;
10977
10978	/* Interrupts are disabled at this point! */
10979	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10980	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10981	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10982	/* Interrupts are re-enabled at this point! */
10983
10984	/*
10985	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10986	*/
10987	if (RT_SUCCESS(rcRun))
10988	{ /* very likely */ }
10989	else
10990	{
10991	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10992	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10993	return rcRun;
10994	}
10995
10996	/*
10997	* Profile the VM-exit.
10998	*/
10999	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11000	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11001	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11002	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11003	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11004	HMVMX_START_EXIT_DISPATCH_PROF();
11005
11006	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11007
11008	/*
11009	* Handle the VM-exit.
11010	*/
11011	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11012	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11013	if (rcStrict == VINF_SUCCESS)
11014	{
11015	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11016	{
11017	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11018	rcStrict = VINF_VMX_VMEXIT;
11019	}
11020	else
11021	{
11022	if (++(*pcLoops) <= cMaxResumeLoops)
11023	continue;
11024	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11025	rcStrict = VINF_EM_RAW_INTERRUPT;
11026	}
11027	}
11028	else
11029	Assert(rcStrict != VINF_VMX_VMEXIT);
11030	break;
11031	}
11032
11033	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11034	return rcStrict;
11035	}
11036	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11037
11038
11039	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11040	* probes.
11041	*
11042	* The following few functions and associated structure contains the bloat
11043	* necessary for providing detailed debug events and dtrace probes as well as
11044	* reliable host side single stepping. This works on the principle of
11045	* "subclassing" the normal execution loop and workers. We replace the loop
11046	* method completely and override selected helpers to add necessary adjustments
11047	* to their core operation.
11048	*
11049	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11050	* any performance for debug and analysis features.
11051	*
11052	* @{
11053	*/
11054
11055	/**
11056	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11057	* the debug run loop.
11058	*/
11059	typedef struct VMXRUNDBGSTATE
11060	{
11061	/** The RIP we started executing at. This is for detecting that we stepped. */
11062	uint64_t uRipStart;
11063	/** The CS we started executing with. */
11064	uint16_t uCsStart;
11065
11066	/** Whether we've actually modified the 1st execution control field. */
11067	bool fModifiedProcCtls : 1;
11068	/** Whether we've actually modified the 2nd execution control field. */
11069	bool fModifiedProcCtls2 : 1;
11070	/** Whether we've actually modified the exception bitmap. */
11071	bool fModifiedXcptBitmap : 1;
11072
11073	/** We desire the modified the CR0 mask to be cleared. */
11074	bool fClearCr0Mask : 1;
11075	/** We desire the modified the CR4 mask to be cleared. */
11076	bool fClearCr4Mask : 1;
11077	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11078	uint32_t fCpe1Extra;
11079	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11080	uint32_t fCpe1Unwanted;
11081	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11082	uint32_t fCpe2Extra;
11083	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11084	uint32_t bmXcptExtra;
11085	/** The sequence number of the Dtrace provider settings the state was
11086	* configured against. */
11087	uint32_t uDtraceSettingsSeqNo;
11088	/** VM-exits to check (one bit per VM-exit). */
11089	uint32_t bmExitsToCheck[3];
11090
11091	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11092	uint32_t fProcCtlsInitial;
11093	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11094	uint32_t fProcCtls2Initial;
11095	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11096	uint32_t bmXcptInitial;
11097	} VMXRUNDBGSTATE;
11098	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11099	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11100
11101
11102	/**
11103	* Initializes the VMXRUNDBGSTATE structure.
11104	*
11105	* @param pVCpu The cross context virtual CPU structure of the
11106	* calling EMT.
11107	* @param pVmxTransient The VMX-transient structure.
11108	* @param pDbgState The debug state to initialize.
11109	*/
11110	static void hmR0VmxRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11111	{
11112	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11113	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11114
11115	pDbgState->fModifiedProcCtls = false;
11116	pDbgState->fModifiedProcCtls2 = false;
11117	pDbgState->fModifiedXcptBitmap = false;
11118	pDbgState->fClearCr0Mask = false;
11119	pDbgState->fClearCr4Mask = false;
11120	pDbgState->fCpe1Extra = 0;
11121	pDbgState->fCpe1Unwanted = 0;
11122	pDbgState->fCpe2Extra = 0;
11123	pDbgState->bmXcptExtra = 0;
11124	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11125	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11126	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11127	}
11128
11129
11130	/**
11131	* Updates the VMSC fields with changes requested by @a pDbgState.
11132	*
11133	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11134	* immediately before executing guest code, i.e. when interrupts are disabled.
11135	* We don't check status codes here as we cannot easily assert or return in the
11136	* latter case.
11137	*
11138	* @param pVCpu The cross context virtual CPU structure.
11139	* @param pVmxTransient The VMX-transient structure.
11140	* @param pDbgState The debug state.
11141	*/
11142	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11143	{
11144	/*
11145	* Ensure desired flags in VMCS control fields are set.
11146	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11147	*
11148	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11149	* there should be no stale data in pCtx at this point.
11150	*/
11151	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11152	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11153	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11154	{
11155	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11156	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11157	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11158	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11159	pDbgState->fModifiedProcCtls = true;
11160	}
11161
11162	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11163	{
11164	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11165	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11166	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11167	pDbgState->fModifiedProcCtls2 = true;
11168	}
11169
11170	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11171	{
11172	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11173	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11174	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11175	pDbgState->fModifiedXcptBitmap = true;
11176	}
11177
11178	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11179	{
11180	pVmcsInfo->u64Cr0Mask = 0;
11181	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, 0);
11182	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11183	}
11184
11185	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11186	{
11187	pVmcsInfo->u64Cr4Mask = 0;
11188	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, 0);
11189	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11190	}
11191
11192	NOREF(pVCpu);
11193	}
11194
11195
11196	/**
11197	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11198	* re-entry next time around.
11199	*
11200	* @returns Strict VBox status code (i.e. informational status codes too).
11201	* @param pVCpu The cross context virtual CPU structure.
11202	* @param pVmxTransient The VMX-transient structure.
11203	* @param pDbgState The debug state.
11204	* @param rcStrict The return code from executing the guest using single
11205	* stepping.
11206	*/
11207	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11208	VBOXSTRICTRC rcStrict)
11209	{
11210	/*
11211	* Restore VM-exit control settings as we may not reenter this function the
11212	* next time around.
11213	*/
11214	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11215
11216	/* We reload the initial value, trigger what we can of recalculations the
11217	next time around. From the looks of things, that's all that's required atm. */
11218	if (pDbgState->fModifiedProcCtls)
11219	{
11220	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11221	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11222	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11223	AssertRC(rc2);
11224	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11225	}
11226
11227	/* We're currently the only ones messing with this one, so just restore the
11228	cached value and reload the field. */
11229	if ( pDbgState->fModifiedProcCtls2
11230	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11231	{
11232	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11233	AssertRC(rc2);
11234	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11235	}
11236
11237	/* If we've modified the exception bitmap, we restore it and trigger
11238	reloading and partial recalculation the next time around. */
11239	if (pDbgState->fModifiedXcptBitmap)
11240	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11241
11242	return rcStrict;
11243	}
11244
11245
11246	/**
11247	* Configures VM-exit controls for current DBGF and DTrace settings.
11248	*
11249	* This updates @a pDbgState and the VMCS execution control fields to reflect
11250	* the necessary VM-exits demanded by DBGF and DTrace.
11251	*
11252	* @param pVCpu The cross context virtual CPU structure.
11253	* @param pVmxTransient The VMX-transient structure. May update
11254	* fUpdatedTscOffsettingAndPreemptTimer.
11255	* @param pDbgState The debug state.
11256	*/
11257	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11258	{
11259	/*
11260	* Take down the dtrace serial number so we can spot changes.
11261	*/
11262	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11263	ASMCompilerBarrier();
11264
11265	/*
11266	* We'll rebuild most of the middle block of data members (holding the
11267	* current settings) as we go along here, so start by clearing it all.
11268	*/
11269	pDbgState->bmXcptExtra = 0;
11270	pDbgState->fCpe1Extra = 0;
11271	pDbgState->fCpe1Unwanted = 0;
11272	pDbgState->fCpe2Extra = 0;
11273	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11274	pDbgState->bmExitsToCheck[i] = 0;
11275
11276	/*
11277	* Software interrupts (INT XXh) - no idea how to trigger these...
11278	*/
11279	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11280	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11281	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11282	{
11283	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11284	}
11285
11286	/*
11287	* INT3 breakpoints - triggered by #BP exceptions.
11288	*/
11289	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11290	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11291
11292	/*
11293	* Exception bitmap and XCPT events+probes.
11294	*/
11295	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11296	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11297	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11298
11299	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11300	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11301	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11302	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11303	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11304	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11305	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11306	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11307	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11308	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11309	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11310	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11311	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11312	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11313	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11314	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11315	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11316	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11317
11318	if (pDbgState->bmXcptExtra)
11319	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11320
11321	/*
11322	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11323	*
11324	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11325	* So, when adding/changing/removing please don't forget to update it.
11326	*
11327	* Some of the macros are picking up local variables to save horizontal space,
11328	* (being able to see it in a table is the lesser evil here).
11329	*/
11330	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11331	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11332	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11333	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11334	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11335	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11336	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11337	} else do { } while (0)
11338	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11339	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11340	{ \
11341	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11342	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11343	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11344	} else do { } while (0)
11345	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11346	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11347	{ \
11348	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11349	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11350	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11351	} else do { } while (0)
11352	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11353	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11354	{ \
11355	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11356	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11357	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11358	} else do { } while (0)
11359
11360	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11361	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11362	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11363	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11364	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11365
11366	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11367	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11368	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11369	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11370	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11371	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11372	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11373	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11374	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11375	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11376	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11377	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11378	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11379	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11380	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11381	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11382	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11383	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11384	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11385	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11386	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11387	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11388	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11389	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11390	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11391	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11392	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11393	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11394	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11395	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11396	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11397	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11398	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11399	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11400	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11401	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11402
11403	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11404	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11405	{
11406	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11407	\| CPUMCTX_EXTRN_APIC_TPR);
11408	AssertRC(rc);
11409
11410	#if 0 /** @todo fix me */
11411	pDbgState->fClearCr0Mask = true;
11412	pDbgState->fClearCr4Mask = true;
11413	#endif
11414	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11415	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11416	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11417	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11418	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11419	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11420	require clearing here and in the loop if we start using it. */
11421	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11422	}
11423	else
11424	{
11425	if (pDbgState->fClearCr0Mask)
11426	{
11427	pDbgState->fClearCr0Mask = false;
11428	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11429	}
11430	if (pDbgState->fClearCr4Mask)
11431	{
11432	pDbgState->fClearCr4Mask = false;
11433	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11434	}
11435	}
11436	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11437	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11438
11439	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11440	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11441	{
11442	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11443	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11444	}
11445	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11446	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11447
11448	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11449	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11450	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11451	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11452	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11453	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11454	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11455	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11456	#if 0 /** @todo too slow, fix handler. */
11457	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11458	#endif
11459	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11460
11461	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11462	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11463	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11464	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11465	{
11466	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11467	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11468	}
11469	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11470	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11471	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11472	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11473
11474	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11475	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11476	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11477	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11478	{
11479	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11480	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11481	}
11482	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11483	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11484	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11485	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11486
11487	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11488	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11489	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11490	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11491	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11492	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11493	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11494	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11495	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11496	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11497	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11498	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11499	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11500	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11501	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11502	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11503	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11504	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11505	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11506	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11507	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11508	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11509
11510	#undef IS_EITHER_ENABLED
11511	#undef SET_ONLY_XBM_IF_EITHER_EN
11512	#undef SET_CPE1_XBM_IF_EITHER_EN
11513	#undef SET_CPEU_XBM_IF_EITHER_EN
11514	#undef SET_CPE2_XBM_IF_EITHER_EN
11515
11516	/*
11517	* Sanitize the control stuff.
11518	*/
11519	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11520	if (pDbgState->fCpe2Extra)
11521	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11522	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11523	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11524	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11525	{
11526	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11527	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11528	}
11529
11530	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11531	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11532	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11533	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11534	}
11535
11536
11537	/**
11538	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11539	* appropriate.
11540	*
11541	* The caller has checked the VM-exit against the
11542	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11543	* already, so we don't have to do that either.
11544	*
11545	* @returns Strict VBox status code (i.e. informational status codes too).
11546	* @param pVCpu The cross context virtual CPU structure.
11547	* @param pVmxTransient The VMX-transient structure.
11548	* @param uExitReason The VM-exit reason.
11549	*
11550	* @remarks The name of this function is displayed by dtrace, so keep it short
11551	* and to the point. No longer than 33 chars long, please.
11552	*/
11553	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11554	{
11555	/*
11556	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11557	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11558	*
11559	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11560	* does. Must add/change/remove both places. Same ordering, please.
11561	*
11562	* Added/removed events must also be reflected in the next section
11563	* where we dispatch dtrace events.
11564	*/
11565	bool fDtrace1 = false;
11566	bool fDtrace2 = false;
11567	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11568	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11569	uint32_t uEventArg = 0;
11570	#define SET_EXIT(a_EventSubName) \
11571	do { \
11572	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11573	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11574	} while (0)
11575	#define SET_BOTH(a_EventSubName) \
11576	do { \
11577	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11578	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11579	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11580	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11581	} while (0)
11582	switch (uExitReason)
11583	{
11584	case VMX_EXIT_MTF:
11585	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11586
11587	case VMX_EXIT_XCPT_OR_NMI:
11588	{
11589	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11590	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11591	{
11592	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11593	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11594	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11595	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11596	{
11597	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11598	{
11599	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11600	uEventArg = pVmxTransient->uExitIntErrorCode;
11601	}
11602	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11603	switch (enmEvent1)
11604	{
11605	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11606	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11607	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11608	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11609	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11610	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11611	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11612	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11613	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11614	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11615	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11616	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11617	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11618	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11619	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11620	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11621	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11622	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11623	default: break;
11624	}
11625	}
11626	else
11627	AssertFailed();
11628	break;
11629
11630	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11631	uEventArg = idxVector;
11632	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11633	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11634	break;
11635	}
11636	break;
11637	}
11638
11639	case VMX_EXIT_TRIPLE_FAULT:
11640	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11641	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11642	break;
11643	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11644	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11645	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11646	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11647	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11648
11649	/* Instruction specific VM-exits: */
11650	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11651	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11652	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11653	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11654	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11655	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11656	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11657	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11658	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11659	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11660	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11661	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11662	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11663	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11664	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11665	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11666	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11667	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11668	case VMX_EXIT_MOV_CRX:
11669	hmR0VmxReadExitQualVmcs(pVmxTransient);
11670	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11671	SET_BOTH(CRX_READ);
11672	else
11673	SET_BOTH(CRX_WRITE);
11674	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11675	break;
11676	case VMX_EXIT_MOV_DRX:
11677	hmR0VmxReadExitQualVmcs(pVmxTransient);
11678	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11679	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11680	SET_BOTH(DRX_READ);
11681	else
11682	SET_BOTH(DRX_WRITE);
11683	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11684	break;
11685	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11686	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11687	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11688	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11689	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11690	case VMX_EXIT_GDTR_IDTR_ACCESS:
11691	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11692	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11693	{
11694	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11695	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11696	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11697	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11698	}
11699	break;
11700
11701	case VMX_EXIT_LDTR_TR_ACCESS:
11702	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11703	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11704	{
11705	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11706	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11707	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11708	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11709	}
11710	break;
11711
11712	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11713	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11714	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11715	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11716	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11717	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11718	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11719	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11720	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11721	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11722	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11723
11724	/* Events that aren't relevant at this point. */
11725	case VMX_EXIT_EXT_INT:
11726	case VMX_EXIT_INT_WINDOW:
11727	case VMX_EXIT_NMI_WINDOW:
11728	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11729	case VMX_EXIT_PREEMPT_TIMER:
11730	case VMX_EXIT_IO_INSTR:
11731	break;
11732
11733	/* Errors and unexpected events. */
11734	case VMX_EXIT_INIT_SIGNAL:
11735	case VMX_EXIT_SIPI:
11736	case VMX_EXIT_IO_SMI:
11737	case VMX_EXIT_SMI:
11738	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11739	case VMX_EXIT_ERR_MSR_LOAD:
11740	case VMX_EXIT_ERR_MACHINE_CHECK:
11741	case VMX_EXIT_PML_FULL:
11742	case VMX_EXIT_VIRTUALIZED_EOI:
11743	break;
11744
11745	default:
11746	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11747	break;
11748	}
11749	#undef SET_BOTH
11750	#undef SET_EXIT
11751
11752	/*
11753	* Dtrace tracepoints go first. We do them here at once so we don't
11754	* have to copy the guest state saving and stuff a few dozen times.
11755	* Down side is that we've got to repeat the switch, though this time
11756	* we use enmEvent since the probes are a subset of what DBGF does.
11757	*/
11758	if (fDtrace1 \|\| fDtrace2)
11759	{
11760	hmR0VmxReadExitQualVmcs(pVmxTransient);
11761	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11762	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11763	switch (enmEvent1)
11764	{
11765	/** @todo consider which extra parameters would be helpful for each probe. */
11766	case DBGFEVENT_END: break;
11767	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11768	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11769	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11770	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11771	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11772	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11773	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11774	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11775	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11776	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11777	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11778	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11779	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11780	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11781	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11782	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11783	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11784	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11785	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11786	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11787	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11788	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11789	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11790	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11791	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11792	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11793	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11794	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11795	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11796	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11797	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11798	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11799	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11800	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11801	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11802	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11803	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11804	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11805	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11806	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11807	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11808	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11809	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11810	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11811	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11812	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11813	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11814	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11815	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11816	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11817	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11818	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11819	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11820	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11821	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11822	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11823	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11824	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11825	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11826	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11827	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11828	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11829	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11830	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11831	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11832	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11833	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11834	}
11835	switch (enmEvent2)
11836	{
11837	/** @todo consider which extra parameters would be helpful for each probe. */
11838	case DBGFEVENT_END: break;
11839	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11840	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11841	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11842	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11843	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11844	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11845	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11846	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11847	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11848	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11849	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11850	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11851	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11852	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11853	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11854	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11855	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11856	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11857	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11858	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11859	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11860	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11861	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11862	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11863	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11864	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11865	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11866	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11867	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11868	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11869	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11870	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11871	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11872	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11873	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11874	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11875	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11876	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11877	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11878	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11879	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11880	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11881	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11882	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11883	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11884	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11885	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11886	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11887	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11888	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11889	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11890	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11891	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11892	}
11893	}
11894
11895	/*
11896	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11897	* the DBGF call will do a full check).
11898	*
11899	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11900	* Note! If we have to events, we prioritize the first, i.e. the instruction
11901	* one, in order to avoid event nesting.
11902	*/
11903	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11904	if ( enmEvent1 != DBGFEVENT_END
11905	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11906	{
11907	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11908	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11909	if (rcStrict != VINF_SUCCESS)
11910	return rcStrict;
11911	}
11912	else if ( enmEvent2 != DBGFEVENT_END
11913	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11914	{
11915	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11916	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11917	if (rcStrict != VINF_SUCCESS)
11918	return rcStrict;
11919	}
11920
11921	return VINF_SUCCESS;
11922	}
11923
11924
11925	/**
11926	* Single-stepping VM-exit filtering.
11927	*
11928	* This is preprocessing the VM-exits and deciding whether we've gotten far
11929	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11930	* handling is performed.
11931	*
11932	* @returns Strict VBox status code (i.e. informational status codes too).
11933	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11934	* @param pVmxTransient The VMX-transient structure.
11935	* @param pDbgState The debug state.
11936	*/
11937	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11938	{
11939	/*
11940	* Expensive (saves context) generic dtrace VM-exit probe.
11941	*/
11942	uint32_t const uExitReason = pVmxTransient->uExitReason;
11943	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11944	{ /* more likely */ }
11945	else
11946	{
11947	hmR0VmxReadExitQualVmcs(pVmxTransient);
11948	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11949	AssertRC(rc);
11950	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11951	}
11952
11953	/*
11954	* Check for host NMI, just to get that out of the way.
11955	*/
11956	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11957	{ /* normally likely */ }
11958	else
11959	{
11960	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11961	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11962	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11963	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
11964	}
11965
11966	/*
11967	* Check for single stepping event if we're stepping.
11968	*/
11969	if (pVCpu->hm.s.fSingleInstruction)
11970	{
11971	switch (uExitReason)
11972	{
11973	case VMX_EXIT_MTF:
11974	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11975
11976	/* Various events: */
11977	case VMX_EXIT_XCPT_OR_NMI:
11978	case VMX_EXIT_EXT_INT:
11979	case VMX_EXIT_TRIPLE_FAULT:
11980	case VMX_EXIT_INT_WINDOW:
11981	case VMX_EXIT_NMI_WINDOW:
11982	case VMX_EXIT_TASK_SWITCH:
11983	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11984	case VMX_EXIT_APIC_ACCESS:
11985	case VMX_EXIT_EPT_VIOLATION:
11986	case VMX_EXIT_EPT_MISCONFIG:
11987	case VMX_EXIT_PREEMPT_TIMER:
11988
11989	/* Instruction specific VM-exits: */
11990	case VMX_EXIT_CPUID:
11991	case VMX_EXIT_GETSEC:
11992	case VMX_EXIT_HLT:
11993	case VMX_EXIT_INVD:
11994	case VMX_EXIT_INVLPG:
11995	case VMX_EXIT_RDPMC:
11996	case VMX_EXIT_RDTSC:
11997	case VMX_EXIT_RSM:
11998	case VMX_EXIT_VMCALL:
11999	case VMX_EXIT_VMCLEAR:
12000	case VMX_EXIT_VMLAUNCH:
12001	case VMX_EXIT_VMPTRLD:
12002	case VMX_EXIT_VMPTRST:
12003	case VMX_EXIT_VMREAD:
12004	case VMX_EXIT_VMRESUME:
12005	case VMX_EXIT_VMWRITE:
12006	case VMX_EXIT_VMXOFF:
12007	case VMX_EXIT_VMXON:
12008	case VMX_EXIT_MOV_CRX:
12009	case VMX_EXIT_MOV_DRX:
12010	case VMX_EXIT_IO_INSTR:
12011	case VMX_EXIT_RDMSR:
12012	case VMX_EXIT_WRMSR:
12013	case VMX_EXIT_MWAIT:
12014	case VMX_EXIT_MONITOR:
12015	case VMX_EXIT_PAUSE:
12016	case VMX_EXIT_GDTR_IDTR_ACCESS:
12017	case VMX_EXIT_LDTR_TR_ACCESS:
12018	case VMX_EXIT_INVEPT:
12019	case VMX_EXIT_RDTSCP:
12020	case VMX_EXIT_INVVPID:
12021	case VMX_EXIT_WBINVD:
12022	case VMX_EXIT_XSETBV:
12023	case VMX_EXIT_RDRAND:
12024	case VMX_EXIT_INVPCID:
12025	case VMX_EXIT_VMFUNC:
12026	case VMX_EXIT_RDSEED:
12027	case VMX_EXIT_XSAVES:
12028	case VMX_EXIT_XRSTORS:
12029	{
12030	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12031	AssertRCReturn(rc, rc);
12032	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12033	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12034	return VINF_EM_DBG_STEPPED;
12035	break;
12036	}
12037
12038	/* Errors and unexpected events: */
12039	case VMX_EXIT_INIT_SIGNAL:
12040	case VMX_EXIT_SIPI:
12041	case VMX_EXIT_IO_SMI:
12042	case VMX_EXIT_SMI:
12043	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12044	case VMX_EXIT_ERR_MSR_LOAD:
12045	case VMX_EXIT_ERR_MACHINE_CHECK:
12046	case VMX_EXIT_PML_FULL:
12047	case VMX_EXIT_VIRTUALIZED_EOI:
12048	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12049	break;
12050
12051	default:
12052	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12053	break;
12054	}
12055	}
12056
12057	/*
12058	* Check for debugger event breakpoints and dtrace probes.
12059	*/
12060	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12061	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12062	{
12063	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12064	if (rcStrict != VINF_SUCCESS)
12065	return rcStrict;
12066	}
12067
12068	/*
12069	* Normal processing.
12070	*/
12071	#ifdef HMVMX_USE_FUNCTION_TABLE
12072	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12073	#else
12074	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12075	#endif
12076	}
12077
12078
12079	/**
12080	* Single steps guest code using hardware-assisted VMX.
12081	*
12082	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12083	* but single-stepping through the hypervisor debugger.
12084	*
12085	* @returns Strict VBox status code (i.e. informational status codes too).
12086	* @param pVCpu The cross context virtual CPU structure.
12087	* @param pcLoops Pointer to the number of executed loops.
12088	*
12089	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12090	*/
12091	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
12092	{
12093	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12094	Assert(pcLoops);
12095	Assert(*pcLoops <= cMaxResumeLoops);
12096
12097	VMXTRANSIENT VmxTransient;
12098	RT_ZERO(VmxTransient);
12099	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12100
12101	/* Set HMCPU indicators. */
12102	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12103	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12104	pVCpu->hm.s.fDebugWantRdTscExit = false;
12105	pVCpu->hm.s.fUsingDebugLoop = true;
12106
12107	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12108	VMXRUNDBGSTATE DbgState;
12109	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12110	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12111
12112	/*
12113	* The loop.
12114	*/
12115	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12116	for (;;)
12117	{
12118	Assert(!HMR0SuspendPending());
12119	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12120	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12121	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12122
12123	/* Set up VM-execution controls the next two can respond to. */
12124	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12125
12126	/*
12127	* Preparatory work for running guest code, this may force us to
12128	* return to ring-3.
12129	*
12130	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12131	*/
12132	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12133	if (rcStrict != VINF_SUCCESS)
12134	break;
12135
12136	/* Interrupts are disabled at this point! */
12137	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12138
12139	/* Override any obnoxious code in the above two calls. */
12140	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12141
12142	/*
12143	* Finally execute the guest.
12144	*/
12145	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12146
12147	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12148	/* Interrupts are re-enabled at this point! */
12149
12150	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12151	if (RT_SUCCESS(rcRun))
12152	{ /* very likely */ }
12153	else
12154	{
12155	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12156	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12157	return rcRun;
12158	}
12159
12160	/* Profile the VM-exit. */
12161	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12162	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12163	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12164	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12165	HMVMX_START_EXIT_DISPATCH_PROF();
12166
12167	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12168
12169	/*
12170	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12171	*/
12172	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12173	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12174	if (rcStrict != VINF_SUCCESS)
12175	break;
12176	if (++(*pcLoops) > cMaxResumeLoops)
12177	{
12178	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12179	rcStrict = VINF_EM_RAW_INTERRUPT;
12180	break;
12181	}
12182
12183	/*
12184	* Stepping: Did the RIP change, if so, consider it a single step.
12185	* Otherwise, make sure one of the TFs gets set.
12186	*/
12187	if (fStepping)
12188	{
12189	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12190	AssertRC(rc);
12191	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12192	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12193	{
12194	rcStrict = VINF_EM_DBG_STEPPED;
12195	break;
12196	}
12197	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12198	}
12199
12200	/*
12201	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12202	*/
12203	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12204	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12205	}
12206
12207	/*
12208	* Clear the X86_EFL_TF if necessary.
12209	*/
12210	if (pVCpu->hm.s.fClearTrapFlag)
12211	{
12212	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12213	AssertRC(rc);
12214	pVCpu->hm.s.fClearTrapFlag = false;
12215	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12216	}
12217	/** @todo there seems to be issues with the resume flag when the monitor trap
12218	* flag is pending without being used. Seen early in bios init when
12219	* accessing APIC page in protected mode. */
12220
12221	/*
12222	* Restore VM-exit control settings as we may not re-enter this function the
12223	* next time around.
12224	*/
12225	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12226
12227	/* Restore HMCPU indicators. */
12228	pVCpu->hm.s.fUsingDebugLoop = false;
12229	pVCpu->hm.s.fDebugWantRdTscExit = false;
12230	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12231
12232	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12233	return rcStrict;
12234	}
12235
12236
12237	/** @} */
12238
12239
12240	/**
12241	* Checks if any expensive dtrace probes are enabled and we should go to the
12242	* debug loop.
12243	*
12244	* @returns true if we should use debug loop, false if not.
12245	*/
12246	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12247	{
12248	/* It's probably faster to OR the raw 32-bit counter variables together.
12249	Since the variables are in an array and the probes are next to one
12250	another (more or less), we have good locality. So, better read
12251	eight-nine cache lines ever time and only have one conditional, than
12252	128+ conditionals, right? */
12253	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12254	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12255	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12256	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12257	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12258	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12259	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12260	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12261	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12262	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12263	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12264	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12265	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12266	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12267	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12268	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12269	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12270	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12271	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12272	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12273	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12274	) != 0
12275	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12276	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12277	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12278	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12279	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12280	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12281	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12282	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12283	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12284	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12285	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12286	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12287	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12288	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12289	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12290	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12291	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12292	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12293	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12294	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12295	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12296	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12297	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12298	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12299	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12300	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12301	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12302	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12303	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12304	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12305	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12306	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12307	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12308	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12309	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12310	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12311	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12312	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12313	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12314	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12315	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12316	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12317	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12318	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12319	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12320	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12321	) != 0
12322	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12323	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12324	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12325	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12326	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12327	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12328	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12329	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12330	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12331	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12332	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12333	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12334	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12335	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12336	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12337	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12338	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12339	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12340	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12341	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12342	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12343	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12344	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12345	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12346	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12347	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12348	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12349	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12350	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12351	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12352	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12353	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12354	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12355	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12356	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12357	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12358	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12359	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12360	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12361	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12362	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12363	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12364	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12365	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12366	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12367	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12368	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12369	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12370	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12371	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12372	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12373	) != 0;
12374	}
12375
12376
12377	/**
12378	* Runs the guest using hardware-assisted VMX.
12379	*
12380	* @returns Strict VBox status code (i.e. informational status codes too).
12381	* @param pVCpu The cross context virtual CPU structure.
12382	*/
12383	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu)
12384	{
12385	AssertPtr(pVCpu);
12386	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12387	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12388	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12389	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12390
12391	VBOXSTRICTRC rcStrict;
12392	uint32_t cLoops = 0;
12393	for (;;)
12394	{
12395	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12396	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12397	#else
12398	NOREF(pCtx);
12399	bool const fInNestedGuestMode = false;
12400	#endif
12401	if (!fInNestedGuestMode)
12402	{
12403	if ( !pVCpu->hm.s.fUseDebugLoop
12404	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12405	&& !DBGFIsStepping(pVCpu)
12406	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12407	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12408	else
12409	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12410	}
12411	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12412	else
12413	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12414
12415	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12416	{
12417	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12418	continue;
12419	}
12420	if (rcStrict == VINF_VMX_VMEXIT)
12421	{
12422	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12423	continue;
12424	}
12425	#endif
12426	break;
12427	}
12428
12429	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12430	switch (rcLoop)
12431	{
12432	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12433	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12434	}
12435
12436	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12437	if (RT_FAILURE(rc2))
12438	{
12439	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12440	rcStrict = rc2;
12441	}
12442	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12443	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12444	return rcStrict;
12445	}
12446
12447
12448	#ifndef HMVMX_USE_FUNCTION_TABLE
12449	/**
12450	* Handles a guest VM-exit from hardware-assisted VMX execution.
12451	*
12452	* @returns Strict VBox status code (i.e. informational status codes too).
12453	* @param pVCpu The cross context virtual CPU structure.
12454	* @param pVmxTransient The VMX-transient structure.
12455	*/
12456	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12457	{
12458	#ifdef DEBUG_ramshankar
12459	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12460	do { \
12461	if (a_fSave != 0) \
12462	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12463	VBOXSTRICTRC rcStrict = a_CallExpr; \
12464	if (a_fSave != 0) \
12465	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12466	return rcStrict; \
12467	} while (0)
12468	#else
12469	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12470	#endif
12471	uint32_t const uExitReason = pVmxTransient->uExitReason;
12472	switch (uExitReason)
12473	{
12474	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12475	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12476	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12477	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12478	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12479	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12480	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12481	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12482	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12483	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12484	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12485	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12486	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12487	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12488	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12489	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12490	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12491	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12492	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12493	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12494	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12495	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12496	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12497	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12498	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12499	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12500	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12501	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12502	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12503	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12504	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12505	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12506	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12507	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12508	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12509	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12510	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12511	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12512	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12513	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12514	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12515	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12516	#else
12517	case VMX_EXIT_VMCLEAR:
12518	case VMX_EXIT_VMLAUNCH:
12519	case VMX_EXIT_VMPTRLD:
12520	case VMX_EXIT_VMPTRST:
12521	case VMX_EXIT_VMREAD:
12522	case VMX_EXIT_VMRESUME:
12523	case VMX_EXIT_VMWRITE:
12524	case VMX_EXIT_VMXOFF:
12525	case VMX_EXIT_VMXON:
12526	case VMX_EXIT_INVVPID:
12527	case VMX_EXIT_INVEPT:
12528	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12529	#endif
12530
12531	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12532	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12533	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12534
12535	case VMX_EXIT_INIT_SIGNAL:
12536	case VMX_EXIT_SIPI:
12537	case VMX_EXIT_IO_SMI:
12538	case VMX_EXIT_SMI:
12539	case VMX_EXIT_ERR_MSR_LOAD:
12540	case VMX_EXIT_ERR_MACHINE_CHECK:
12541	case VMX_EXIT_PML_FULL:
12542	case VMX_EXIT_VIRTUALIZED_EOI:
12543	case VMX_EXIT_GDTR_IDTR_ACCESS:
12544	case VMX_EXIT_LDTR_TR_ACCESS:
12545	case VMX_EXIT_APIC_WRITE:
12546	case VMX_EXIT_RDRAND:
12547	case VMX_EXIT_RSM:
12548	case VMX_EXIT_VMFUNC:
12549	case VMX_EXIT_ENCLS:
12550	case VMX_EXIT_RDSEED:
12551	case VMX_EXIT_XSAVES:
12552	case VMX_EXIT_XRSTORS:
12553	case VMX_EXIT_UMWAIT:
12554	case VMX_EXIT_TPAUSE:
12555	default:
12556	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12557	}
12558	#undef VMEXIT_CALL_RET
12559	}
12560	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12561
12562
12563	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12564	/**
12565	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12566	*
12567	* @returns Strict VBox status code (i.e. informational status codes too).
12568	* @param pVCpu The cross context virtual CPU structure.
12569	* @param pVmxTransient The VMX-transient structure.
12570	*/
12571	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12572	{
12573	/** @todo NSTVMX: Remove after debugging page-fault issue. */
12574	#ifdef DEBUG_ramshankar
12575	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12576	Log4Func(("cs:rip=%#04x:%#RX64 rsp=%#RX64 eflags=%#RX32 cr3=%#RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12577	pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.cr3));
12578	#endif
12579
12580	uint32_t const uExitReason = pVmxTransient->uExitReason;
12581	switch (uExitReason)
12582	{
12583	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12584	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12585	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12586	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12587	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12588
12589	/*
12590	* We shouldn't direct host physical interrupts to the nested-guest.
12591	*/
12592	case VMX_EXIT_EXT_INT:
12593	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12594
12595	/*
12596	* Instructions that cause VM-exits unconditionally or the condition is
12597	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12598	* happens, it's guaranteed to be a nested-guest VM-exit).
12599	*
12600	* - Provides VM-exit instruction length ONLY.
12601	*/
12602	case VMX_EXIT_CPUID: /* Unconditional. */
12603	case VMX_EXIT_VMCALL:
12604	case VMX_EXIT_GETSEC:
12605	case VMX_EXIT_INVD:
12606	case VMX_EXIT_XSETBV:
12607	case VMX_EXIT_VMLAUNCH:
12608	case VMX_EXIT_VMRESUME:
12609	case VMX_EXIT_VMXOFF:
12610	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
12611	case VMX_EXIT_VMFUNC:
12612	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12613
12614	/*
12615	* Instructions that cause VM-exits unconditionally or the condition is
12616	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12617	* happens, it's guaranteed to be a nested-guest VM-exit).
12618	*
12619	* - Provides VM-exit instruction length.
12620	* - Provides VM-exit information.
12621	* - Optionally provides Exit qualification.
12622	*
12623	* Since Exit qualification is 0 for all VM-exits where it is not
12624	* applicable, reading and passing it to the guest should produce
12625	* defined behavior.
12626	*
12627	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12628	*/
12629	case VMX_EXIT_INVEPT: /* Unconditional. */
12630	case VMX_EXIT_INVVPID:
12631	case VMX_EXIT_VMCLEAR:
12632	case VMX_EXIT_VMPTRLD:
12633	case VMX_EXIT_VMPTRST:
12634	case VMX_EXIT_VMXON:
12635	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
12636	case VMX_EXIT_LDTR_TR_ACCESS:
12637	case VMX_EXIT_RDRAND:
12638	case VMX_EXIT_RDSEED:
12639	case VMX_EXIT_XSAVES:
12640	case VMX_EXIT_XRSTORS:
12641	case VMX_EXIT_UMWAIT:
12642	case VMX_EXIT_TPAUSE:
12643	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12644
12645	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12646	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12647	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12648	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12649	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12650	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12651	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12652	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12653	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12654	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12655	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12656	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12657	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12658	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12659	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12660	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12661	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12662	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12663	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12664
12665	case VMX_EXIT_PREEMPT_TIMER:
12666	{
12667	/** @todo NSTVMX: Preempt timer. */
12668	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12669	}
12670
12671	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12672	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12673
12674	case VMX_EXIT_VMREAD:
12675	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12676
12677	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12678	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12679
12680	case VMX_EXIT_INIT_SIGNAL:
12681	case VMX_EXIT_SIPI:
12682	case VMX_EXIT_IO_SMI:
12683	case VMX_EXIT_SMI:
12684	case VMX_EXIT_ERR_MSR_LOAD:
12685	case VMX_EXIT_ERR_MACHINE_CHECK:
12686	case VMX_EXIT_PML_FULL:
12687	case VMX_EXIT_RSM:
12688	default:
12689	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12690	}
12691	}
12692	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12693
12694
12695	/** @name VM-exit helpers.
12696	* @{
12697	*/
12698	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12699	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12700	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12701
12702	/** Macro for VM-exits called unexpectedly. */
12703	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
12704	do { \
12705	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
12706	return VERR_VMX_UNEXPECTED_EXIT; \
12707	} while (0)
12708
12709	#ifdef VBOX_STRICT
12710	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12711	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12712	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12713
12714	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12715	do { \
12716	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12717	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12718	} while (0)
12719
12720	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12721	do { \
12722	AssertPtr((a_pVCpu)); \
12723	AssertPtr((a_pVmxTransient)); \
12724	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12725	Assert((a_pVmxTransient)->pVmcsInfo); \
12726	Assert(ASMIntAreEnabled()); \
12727	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12728	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12729	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
12730	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12731	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12732	HMVMX_ASSERT_PREEMPT_CPUID(); \
12733	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12734	} while (0)
12735
12736	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12737	do { \
12738	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12739	Assert((a_pVmxTransient)->fIsNestedGuest); \
12740	} while (0)
12741
12742	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12743	do { \
12744	Log4Func(("\n")); \
12745	} while (0)
12746	#else
12747	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12748	do { \
12749	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12750	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12751	} while (0)
12752
12753	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12754	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12755
12756	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12757	#endif
12758
12759	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12760	/** Macro that does the necessary privilege checks and intercepted VM-exits for
12761	* guests that attempted to execute a VMX instruction. */
12762	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
12763	do \
12764	{ \
12765	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
12766	if (rcStrictTmp == VINF_SUCCESS) \
12767	{ /* likely */ } \
12768	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12769	{ \
12770	Assert((a_pVCpu)->hm.s.Event.fPending); \
12771	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
12772	return VINF_SUCCESS; \
12773	} \
12774	else \
12775	{ \
12776	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
12777	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
12778	} \
12779	} while (0)
12780
12781	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
12782	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
12783	do \
12784	{ \
12785	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
12786	(a_pGCPtrEffAddr)); \
12787	if (rcStrictTmp == VINF_SUCCESS) \
12788	{ /* likely */ } \
12789	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12790	{ \
12791	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
12792	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
12793	NOREF(uXcptTmp); \
12794	return VINF_SUCCESS; \
12795	} \
12796	else \
12797	{ \
12798	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
12799	return rcStrictTmp; \
12800	} \
12801	} while (0)
12802	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12803
12804
12805	/**
12806	* Advances the guest RIP by the specified number of bytes.
12807	*
12808	* @param pVCpu The cross context virtual CPU structure.
12809	* @param cbInstr Number of bytes to advance the RIP by.
12810	*
12811	* @remarks No-long-jump zone!!!
12812	*/
12813	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
12814	{
12815	/* Advance the RIP. */
12816	pVCpu->cpum.GstCtx.rip += cbInstr;
12817	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12818
12819	/* Update interrupt inhibition. */
12820	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12821	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12822	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12823	}
12824
12825
12826	/**
12827	* Advances the guest RIP after reading it from the VMCS.
12828	*
12829	* @returns VBox status code, no informational status codes.
12830	* @param pVCpu The cross context virtual CPU structure.
12831	* @param pVmxTransient The VMX-transient structure.
12832	*
12833	* @remarks No-long-jump zone!!!
12834	*/
12835	static int hmR0VmxAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12836	{
12837	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12838	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12839	AssertRCReturn(rc, rc);
12840
12841	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
12842	return VINF_SUCCESS;
12843	}
12844
12845
12846	/**
12847	* Handle a condition that occurred while delivering an event through the guest or
12848	* nested-guest IDT.
12849	*
12850	* @returns Strict VBox status code (i.e. informational status codes too).
12851	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12852	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12853	* to continue execution of the guest which will delivery the \#DF.
12854	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12855	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12856	*
12857	* @param pVCpu The cross context virtual CPU structure.
12858	* @param pVmxTransient The VMX-transient structure.
12859	*
12860	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
12861	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
12862	* is due to an EPT violation, PML full or SPP-related event.
12863	*
12864	* @remarks No-long-jump zone!!!
12865	*/
12866	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12867	{
12868	Assert(!pVCpu->hm.s.Event.fPending);
12869	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
12870	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
12871	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
12872	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
12873	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
12874
12875	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12876	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12877	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
12878	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12879	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
12880	{
12881	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
12882	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
12883
12884	/*
12885	* If the event was a software interrupt (generated with INT n) or a software exception
12886	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12887	* can handle the VM-exit and continue guest execution which will re-execute the
12888	* instruction rather than re-injecting the exception, as that can cause premature
12889	* trips to ring-3 before injection and involve TRPM which currently has no way of
12890	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12891	* the problem).
12892	*/
12893	IEMXCPTRAISE enmRaise;
12894	IEMXCPTRAISEINFO fRaiseInfo;
12895	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12896	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12897	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12898	{
12899	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12900	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12901	}
12902	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
12903	{
12904	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
12905	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12906	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
12907
12908	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12909	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12910
12911	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12912
12913	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12914	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12915	{
12916	pVmxTransient->fVectoringPF = true;
12917	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12918	}
12919	}
12920	else
12921	{
12922	/*
12923	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12924	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12925	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12926	*/
12927	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12928	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12929	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12930	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12931	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12932	}
12933
12934	/*
12935	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12936	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12937	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12938	* subsequent VM-entry would fail, see @bugref{7445}.
12939	*
12940	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
12941	*/
12942	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12943	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
12944	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
12945	&& CPUMIsGuestNmiBlocking(pVCpu))
12946	{
12947	CPUMSetGuestNmiBlocking(pVCpu, false);
12948	}
12949
12950	switch (enmRaise)
12951	{
12952	case IEMXCPTRAISE_CURRENT_XCPT:
12953	{
12954	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
12955	Assert(rcStrict == VINF_SUCCESS);
12956	break;
12957	}
12958
12959	case IEMXCPTRAISE_PREV_EVENT:
12960	{
12961	uint32_t u32ErrCode;
12962	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
12963	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12964	else
12965	u32ErrCode = 0;
12966
12967	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12968	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
12969	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
12970	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12971
12972	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12973	pVCpu->hm.s.Event.u32ErrCode));
12974	Assert(rcStrict == VINF_SUCCESS);
12975	break;
12976	}
12977
12978	case IEMXCPTRAISE_REEXEC_INSTR:
12979	Assert(rcStrict == VINF_SUCCESS);
12980	break;
12981
12982	case IEMXCPTRAISE_DOUBLE_FAULT:
12983	{
12984	/*
12985	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12986	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12987	*/
12988	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12989	{
12990	pVmxTransient->fVectoringDoublePF = true;
12991	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12992	pVCpu->cpum.GstCtx.cr2));
12993	rcStrict = VINF_SUCCESS;
12994	}
12995	else
12996	{
12997	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
12998	hmR0VmxSetPendingXcptDF(pVCpu);
12999	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13000	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13001	rcStrict = VINF_HM_DOUBLE_FAULT;
13002	}
13003	break;
13004	}
13005
13006	case IEMXCPTRAISE_TRIPLE_FAULT:
13007	{
13008	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13009	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13010	rcStrict = VINF_EM_RESET;
13011	break;
13012	}
13013
13014	case IEMXCPTRAISE_CPU_HANG:
13015	{
13016	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13017	rcStrict = VERR_EM_GUEST_CPU_HANG;
13018	break;
13019	}
13020
13021	default:
13022	{
13023	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13024	rcStrict = VERR_VMX_IPE_2;
13025	break;
13026	}
13027	}
13028	}
13029	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13030	&& !CPUMIsGuestNmiBlocking(pVCpu))
13031	{
13032	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13033	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13034	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13035	{
13036	/*
13037	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13038	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13039	* that NMIs remain blocked until the IRET execution is completed.
13040	*
13041	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13042	*/
13043	CPUMSetGuestNmiBlocking(pVCpu, true);
13044	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13045	}
13046	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13047	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13048	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13049	{
13050	/*
13051	* Execution of IRET caused an EPT violation, page-modification log-full event or
13052	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13053	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13054	* that NMIs remain blocked until the IRET execution is completed.
13055	*
13056	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13057	*/
13058	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13059	{
13060	CPUMSetGuestNmiBlocking(pVCpu, true);
13061	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13062	}
13063	}
13064	}
13065
13066	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13067	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13068	return rcStrict;
13069	}
13070
13071
13072	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13073	/**
13074	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13075	* guest attempting to execute a VMX instruction.
13076	*
13077	* @returns Strict VBox status code (i.e. informational status codes too).
13078	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13079	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13080	*
13081	* @param pVCpu The cross context virtual CPU structure.
13082	* @param uExitReason The VM-exit reason.
13083	*
13084	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13085	* @remarks No-long-jump zone!!!
13086	*/
13087	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
13088	{
13089	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13090	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13091
13092	/*
13093	* The physical CPU would have already checked the CPU mode/code segment.
13094	* We shall just assert here for paranoia.
13095	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13096	*/
13097	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13098	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13099	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13100
13101	if (uExitReason == VMX_EXIT_VMXON)
13102	{
13103	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13104
13105	/*
13106	* We check CR4.VMXE because it is required to be always set while in VMX operation
13107	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13108	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13109	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13110	*/
13111	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13112	{
13113	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13114	hmR0VmxSetPendingXcptUD(pVCpu);
13115	return VINF_HM_PENDING_XCPT;
13116	}
13117	}
13118	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13119	{
13120	/*
13121	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13122	* (other than VMXON), we need to raise a #UD.
13123	*/
13124	Log4Func(("Not in VMX root mode -> #UD\n"));
13125	hmR0VmxSetPendingXcptUD(pVCpu);
13126	return VINF_HM_PENDING_XCPT;
13127	}
13128
13129	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13130	return VINF_SUCCESS;
13131	}
13132
13133
13134	/**
13135	* Decodes the memory operand of an instruction that caused a VM-exit.
13136	*
13137	* The Exit qualification field provides the displacement field for memory
13138	* operand instructions, if any.
13139	*
13140	* @returns Strict VBox status code (i.e. informational status codes too).
13141	* @retval VINF_SUCCESS if the operand was successfully decoded.
13142	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13143	* operand.
13144	* @param pVCpu The cross context virtual CPU structure.
13145	* @param uExitInstrInfo The VM-exit instruction information field.
13146	* @param enmMemAccess The memory operand's access type (read or write).
13147	* @param GCPtrDisp The instruction displacement field, if any. For
13148	* RIP-relative addressing pass RIP + displacement here.
13149	* @param pGCPtrMem Where to store the effective destination memory address.
13150	*
13151	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13152	* virtual-8086 mode hence skips those checks while verifying if the
13153	* segment is valid.
13154	*/
13155	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13156	PRTGCPTR pGCPtrMem)
13157	{
13158	Assert(pGCPtrMem);
13159	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13160	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13161	\| CPUMCTX_EXTRN_CR0);
13162
13163	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13164	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13165	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13166
13167	VMXEXITINSTRINFO ExitInstrInfo;
13168	ExitInstrInfo.u = uExitInstrInfo;
13169	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13170	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13171	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13172	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13173	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13174	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13175	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13176	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13177	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13178
13179	/*
13180	* Validate instruction information.
13181	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13182	*/
13183	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13184	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13185	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13186	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13187	AssertLogRelMsgReturn(fIsMemOperand,
13188	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13189
13190	/*
13191	* Compute the complete effective address.
13192	*
13193	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13194	* See AMD spec. 4.5.2 "Segment Registers".
13195	*/
13196	RTGCPTR GCPtrMem = GCPtrDisp;
13197	if (fBaseRegValid)
13198	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13199	if (fIdxRegValid)
13200	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13201
13202	RTGCPTR const GCPtrOff = GCPtrMem;
13203	if ( !fIsLongMode
13204	\|\| iSegReg >= X86_SREG_FS)
13205	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13206	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13207
13208	/*
13209	* Validate effective address.
13210	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13211	*/
13212	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13213	Assert(cbAccess > 0);
13214	if (fIsLongMode)
13215	{
13216	if (X86_IS_CANONICAL(GCPtrMem))
13217	{
13218	*pGCPtrMem = GCPtrMem;
13219	return VINF_SUCCESS;
13220	}
13221
13222	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13223	* "Data Limit Checks in 64-bit Mode". */
13224	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13225	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13226	return VINF_HM_PENDING_XCPT;
13227	}
13228
13229	/*
13230	* This is a watered down version of iemMemApplySegment().
13231	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13232	* and segment CPL/DPL checks are skipped.
13233	*/
13234	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13235	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13236	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13237
13238	/* Check if the segment is present and usable. */
13239	if ( pSel->Attr.n.u1Present
13240	&& !pSel->Attr.n.u1Unusable)
13241	{
13242	Assert(pSel->Attr.n.u1DescType);
13243	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13244	{
13245	/* Check permissions for the data segment. */
13246	if ( enmMemAccess == VMXMEMACCESS_WRITE
13247	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13248	{
13249	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13250	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13251	return VINF_HM_PENDING_XCPT;
13252	}
13253
13254	/* Check limits if it's a normal data segment. */
13255	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13256	{
13257	if ( GCPtrFirst32 > pSel->u32Limit
13258	\|\| GCPtrLast32 > pSel->u32Limit)
13259	{
13260	Log4Func(("Data segment limit exceeded. "
13261	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13262	GCPtrLast32, pSel->u32Limit));
13263	if (iSegReg == X86_SREG_SS)
13264	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13265	else
13266	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13267	return VINF_HM_PENDING_XCPT;
13268	}
13269	}
13270	else
13271	{
13272	/* Check limits if it's an expand-down data segment.
13273	Note! The upper boundary is defined by the B bit, not the G bit! */
13274	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13275	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13276	{
13277	Log4Func(("Expand-down data segment limit exceeded. "
13278	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13279	GCPtrLast32, pSel->u32Limit));
13280	if (iSegReg == X86_SREG_SS)
13281	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13282	else
13283	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13284	return VINF_HM_PENDING_XCPT;
13285	}
13286	}
13287	}
13288	else
13289	{
13290	/* Check permissions for the code segment. */
13291	if ( enmMemAccess == VMXMEMACCESS_WRITE
13292	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13293	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13294	{
13295	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13296	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13297	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13298	return VINF_HM_PENDING_XCPT;
13299	}
13300
13301	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13302	if ( GCPtrFirst32 > pSel->u32Limit
13303	\|\| GCPtrLast32 > pSel->u32Limit)
13304	{
13305	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13306	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13307	if (iSegReg == X86_SREG_SS)
13308	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13309	else
13310	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13311	return VINF_HM_PENDING_XCPT;
13312	}
13313	}
13314	}
13315	else
13316	{
13317	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13318	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13319	return VINF_HM_PENDING_XCPT;
13320	}
13321
13322	*pGCPtrMem = GCPtrMem;
13323	return VINF_SUCCESS;
13324	}
13325	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13326
13327
13328	/**
13329	* VM-exit helper for LMSW.
13330	*/
13331	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13332	{
13333	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13334	AssertRCReturn(rc, rc);
13335
13336	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13337	AssertMsg( rcStrict == VINF_SUCCESS
13338	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13339
13340	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13341	if (rcStrict == VINF_IEM_RAISED_XCPT)
13342	{
13343	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13344	rcStrict = VINF_SUCCESS;
13345	}
13346
13347	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13348	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13349	return rcStrict;
13350	}
13351
13352
13353	/**
13354	* VM-exit helper for CLTS.
13355	*/
13356	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13357	{
13358	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13359	AssertRCReturn(rc, rc);
13360
13361	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13362	AssertMsg( rcStrict == VINF_SUCCESS
13363	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13364
13365	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13366	if (rcStrict == VINF_IEM_RAISED_XCPT)
13367	{
13368	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13369	rcStrict = VINF_SUCCESS;
13370	}
13371
13372	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13373	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13374	return rcStrict;
13375	}
13376
13377
13378	/**
13379	* VM-exit helper for MOV from CRx (CRx read).
13380	*/
13381	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13382	{
13383	Assert(iCrReg < 16);
13384	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13385
13386	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13387	AssertRCReturn(rc, rc);
13388
13389	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13390	AssertMsg( rcStrict == VINF_SUCCESS
13391	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13392
13393	if (iGReg == X86_GREG_xSP)
13394	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13395	else
13396	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13397	#ifdef VBOX_WITH_STATISTICS
13398	switch (iCrReg)
13399	{
13400	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13401	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13402	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13403	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13404	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13405	}
13406	#endif
13407	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13408	return rcStrict;
13409	}
13410
13411
13412	/**
13413	* VM-exit helper for MOV to CRx (CRx write).
13414	*/
13415	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13416	{
13417	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13418	AssertRCReturn(rc, rc);
13419
13420	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13421	AssertMsg( rcStrict == VINF_SUCCESS
13422	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13423	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13424
13425	switch (iCrReg)
13426	{
13427	case 0:
13428	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
13429	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13430	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13431	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13432	break;
13433
13434	case 2:
13435	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13436	/* Nothing to do here, CR2 it's not part of the VMCS. */
13437	break;
13438
13439	case 3:
13440	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13441	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13442	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13443	break;
13444
13445	case 4:
13446	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13447	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13448	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13449	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13450	break;
13451
13452	case 8:
13453	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13454	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13456	break;
13457
13458	default:
13459	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13460	break;
13461	}
13462
13463	if (rcStrict == VINF_IEM_RAISED_XCPT)
13464	{
13465	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13466	rcStrict = VINF_SUCCESS;
13467	}
13468	return rcStrict;
13469	}
13470
13471
13472	/**
13473	* VM-exit exception handler for \#PF (Page-fault exception).
13474	*
13475	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13476	*/
13477	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13478	{
13479	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13480	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13481	hmR0VmxReadExitQualVmcs(pVmxTransient);
13482
13483	if (!pVM->hm.s.fNestedPaging)
13484	{ /* likely */ }
13485	else
13486	{
13487	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13488	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13489	#endif
13490	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13491	if (!pVmxTransient->fVectoringDoublePF)
13492	{
13493	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13494	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13495	}
13496	else
13497	{
13498	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13499	Assert(!pVmxTransient->fIsNestedGuest);
13500	hmR0VmxSetPendingXcptDF(pVCpu);
13501	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13502	}
13503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13504	return VINF_SUCCESS;
13505	}
13506
13507	Assert(!pVmxTransient->fIsNestedGuest);
13508
13509	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13510	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13511	if (pVmxTransient->fVectoringPF)
13512	{
13513	Assert(pVCpu->hm.s.Event.fPending);
13514	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13515	}
13516
13517	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13518	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13519	AssertRCReturn(rc, rc);
13520
13521	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
13522	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
13523
13524	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13525	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13526
13527	Log4Func(("#PF: rc=%Rrc\n", rc));
13528	if (rc == VINF_SUCCESS)
13529	{
13530	/*
13531	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13532	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13533	*/
13534	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13535	TRPMResetTrap(pVCpu);
13536	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13537	return rc;
13538	}
13539
13540	if (rc == VINF_EM_RAW_GUEST_TRAP)
13541	{
13542	if (!pVmxTransient->fVectoringDoublePF)
13543	{
13544	/* It's a guest page fault and needs to be reflected to the guest. */
13545	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
13546	TRPMResetTrap(pVCpu);
13547	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13548	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13549	uGstErrorCode, pVmxTransient->uExitQual);
13550	}
13551	else
13552	{
13553	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13554	TRPMResetTrap(pVCpu);
13555	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13556	hmR0VmxSetPendingXcptDF(pVCpu);
13557	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13558	}
13559
13560	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13561	return VINF_SUCCESS;
13562	}
13563
13564	TRPMResetTrap(pVCpu);
13565	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13566	return rc;
13567	}
13568
13569
13570	/**
13571	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13572	*
13573	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13574	*/
13575	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13576	{
13577	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13578	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13579
13580	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13581	AssertRCReturn(rc, rc);
13582
13583	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13584	{
13585	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13586	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13587
13588	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13589	* provides VM-exit instruction length. If this causes problem later,
13590	* disassemble the instruction like it's done on AMD-V. */
13591	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13592	AssertRCReturn(rc2, rc2);
13593	return rc;
13594	}
13595
13596	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
13597	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13598	return VINF_SUCCESS;
13599	}
13600
13601
13602	/**
13603	* VM-exit exception handler for \#BP (Breakpoint exception).
13604	*
13605	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13606	*/
13607	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13608	{
13609	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13611
13612	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13613	AssertRCReturn(rc, rc);
13614
13615	if (!pVmxTransient->fIsNestedGuest)
13616	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13617	else
13618	rc = VINF_EM_RAW_GUEST_TRAP;
13619
13620	if (rc == VINF_EM_RAW_GUEST_TRAP)
13621	{
13622	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13623	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13624	rc = VINF_SUCCESS;
13625	}
13626
13627	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
13628	return rc;
13629	}
13630
13631
13632	/**
13633	* VM-exit exception handler for \#AC (Alignment-check exception).
13634	*
13635	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13636	*/
13637	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13638	{
13639	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13641
13642	/* Re-inject it. We'll detect any nesting before getting here. */
13643	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13644	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13645	return VINF_SUCCESS;
13646	}
13647
13648
13649	/**
13650	* VM-exit exception handler for \#DB (Debug exception).
13651	*
13652	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13653	*/
13654	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13655	{
13656	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13657	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13658
13659	/*
13660	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13661	*/
13662	hmR0VmxReadExitQualVmcs(pVmxTransient);
13663
13664	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13665	uint64_t const uDR6 = X86_DR6_INIT_VAL
13666	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13667	\| X86_DR6_BD \| X86_DR6_BS));
13668
13669	int rc;
13670	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13671	if (!pVmxTransient->fIsNestedGuest)
13672	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13673	else
13674	rc = VINF_EM_RAW_GUEST_TRAP;
13675	Log6Func(("rc=%Rrc\n", rc));
13676	if (rc == VINF_EM_RAW_GUEST_TRAP)
13677	{
13678	/*
13679	* The exception was for the guest. Update DR6, DR7.GD and
13680	* IA32_DEBUGCTL.LBR before forwarding it.
13681	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
13682	*/
13683	VMMRZCallRing3Disable(pVCpu);
13684	HM_DISABLE_PREEMPT(pVCpu);
13685
13686	pCtx->dr[6] &= ~X86_DR6_B_MASK;
13687	pCtx->dr[6] \|= uDR6;
13688	if (CPUMIsGuestDebugStateActive(pVCpu))
13689	ASMSetDR6(pCtx->dr[6]);
13690
13691	HM_RESTORE_PREEMPT();
13692	VMMRZCallRing3Enable(pVCpu);
13693
13694	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
13695	AssertRCReturn(rc, rc);
13696
13697	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13698	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
13699
13700	/* Paranoia. */
13701	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
13702	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
13703
13704	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
13705	AssertRC(rc);
13706
13707	/*
13708	* Raise #DB in the guest.
13709	*
13710	* It is important to reflect exactly what the VM-exit gave us (preserving the
13711	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13712	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13713	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13714	*
13715	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13716	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13717	*/
13718	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13719	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13720	return VINF_SUCCESS;
13721	}
13722
13723	/*
13724	* Not a guest trap, must be a hypervisor related debug event then.
13725	* Update DR6 in case someone is interested in it.
13726	*/
13727	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13728	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13729	CPUMSetHyperDR6(pVCpu, uDR6);
13730
13731	return rc;
13732	}
13733
13734
13735	/**
13736	* Hacks its way around the lovely mesa driver's backdoor accesses.
13737	*
13738	* @sa hmR0SvmHandleMesaDrvGp.
13739	*/
13740	static int hmR0VmxHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13741	{
13742	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
13743	RT_NOREF(pCtx);
13744
13745	/* For now we'll just skip the instruction. */
13746	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13747	}
13748
13749
13750	/**
13751	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
13752	* backdoor logging w/o checking what it is running inside.
13753	*
13754	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
13755	* backdoor port and magic numbers loaded in registers.
13756	*
13757	* @returns true if it is, false if it isn't.
13758	* @sa hmR0SvmIsMesaDrvGp.
13759	*/
13760	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13761	{
13762	/* 0xed: IN eAX,dx */
13763	uint8_t abInstr[1];
13764	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
13765	return false;
13766
13767	/* Check that it is #GP(0). */
13768	if (pVmxTransient->uExitIntErrorCode != 0)
13769	return false;
13770
13771	/* Check magic and port. */
13772	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
13773	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
13774	if (pCtx->rax != UINT32_C(0x564d5868))
13775	return false;
13776	if (pCtx->dx != UINT32_C(0x5658))
13777	return false;
13778
13779	/* Flat ring-3 CS. */
13780	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
13781	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
13782	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
13783	if (pCtx->cs.Attr.n.u2Dpl != 3)
13784	return false;
13785	if (pCtx->cs.u64Base != 0)
13786	return false;
13787
13788	/* Check opcode. */
13789	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
13790	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
13791	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
13792	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
13793	if (RT_FAILURE(rc))
13794	return false;
13795	if (abInstr[0] != 0xed)
13796	return false;
13797
13798	return true;
13799	}
13800
13801
13802	/**
13803	* VM-exit exception handler for \#GP (General-protection exception).
13804	*
13805	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13806	*/
13807	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13808	{
13809	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13811
13812	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13813	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13814	if (pVmcsInfo->RealMode.fRealOnV86Active)
13815	{ /* likely */ }
13816	else
13817	{
13818	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13819	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
13820	#endif
13821	/*
13822	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
13823	* executing a nested-guest, reflect #GP to the guest or nested-guest.
13824	*/
13825	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13826	AssertRCReturn(rc, rc);
13827	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13828	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13829
13830	if ( pVmxTransient->fIsNestedGuest
13831	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
13832	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
13833	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13834	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13835	else
13836	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
13837	return rc;
13838	}
13839
13840	Assert(CPUMIsGuestInRealModeEx(pCtx));
13841	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13842	Assert(!pVmxTransient->fIsNestedGuest);
13843
13844	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13845	AssertRCReturn(rc, rc);
13846
13847	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13848	if (rcStrict == VINF_SUCCESS)
13849	{
13850	if (!CPUMIsGuestInRealModeEx(pCtx))
13851	{
13852	/*
13853	* The guest is no longer in real-mode, check if we can continue executing the
13854	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13855	*/
13856	pVmcsInfo->RealMode.fRealOnV86Active = false;
13857	if (HMCanExecuteVmxGuest(pVCpu->pVMR0, pVCpu, pCtx))
13858	{
13859	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
13860	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13861	}
13862	else
13863	{
13864	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13865	rcStrict = VINF_EM_RESCHEDULE;
13866	}
13867	}
13868	else
13869	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13870	}
13871	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13872	{
13873	rcStrict = VINF_SUCCESS;
13874	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13875	}
13876	return VBOXSTRICTRC_VAL(rcStrict);
13877	}
13878
13879
13880	/**
13881	* VM-exit exception handler wrapper for all other exceptions that are not handled
13882	* by a specific handler.
13883	*
13884	* This simply re-injects the exception back into the VM without any special
13885	* processing.
13886	*
13887	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13888	*/
13889	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13890	{
13891	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13892
13893	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13894	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13895	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
13896	("uVector=%#x u32XcptBitmap=%#X32\n",
13897	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
13898	NOREF(pVmcsInfo);
13899	#endif
13900
13901	/*
13902	* Re-inject the exception into the guest. This cannot be a double-fault condition which
13903	* would have been handled while checking exits due to event delivery.
13904	*/
13905	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13906
13907	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13908	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13909	AssertRCReturn(rc, rc);
13910	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13911	#endif
13912
13913	#ifdef VBOX_WITH_STATISTICS
13914	switch (uVector)
13915	{
13916	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
13917	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
13918	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
13919	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
13920	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
13921	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
13922	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
13923	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
13924	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
13925	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
13926	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
13927	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
13928	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
13929	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
13930	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
13931	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
13932	default:
13933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13934	break;
13935	}
13936	#endif
13937
13938	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
13939	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
13940	NOREF(uVector);
13941
13942	/* Re-inject the original exception into the guest. */
13943	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13944	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13945	return VINF_SUCCESS;
13946	}
13947
13948
13949	/**
13950	* VM-exit exception handler for all exceptions (except NMIs!).
13951	*
13952	* @remarks This may be called for both guests and nested-guests. Take care to not
13953	* make assumptions and avoid doing anything that is not relevant when
13954	* executing a nested-guest (e.g., Mesa driver hacks).
13955	*/
13956	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13957	{
13958	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
13959
13960	/*
13961	* If this VM-exit occurred while delivering an event through the guest IDT, take
13962	* action based on the return code and additional hints (e.g. for page-faults)
13963	* that will be updated in the VMX transient structure.
13964	*/
13965	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13966	if (rcStrict == VINF_SUCCESS)
13967	{
13968	/*
13969	* If an exception caused a VM-exit due to delivery of an event, the original
13970	* event may have to be re-injected into the guest. We shall reinject it and
13971	* continue guest execution. However, page-fault is a complicated case and
13972	* needs additional processing done in hmR0VmxExitXcptPF().
13973	*/
13974	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13975	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13976	if ( !pVCpu->hm.s.Event.fPending
13977	\|\| uVector == X86_XCPT_PF)
13978	{
13979	switch (uVector)
13980	{
13981	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
13982	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
13983	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
13984	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
13985	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
13986	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
13987	default:
13988	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
13989	}
13990	}
13991	/* else: inject pending event before resuming guest execution. */
13992	}
13993	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
13994	{
13995	Assert(pVCpu->hm.s.Event.fPending);
13996	rcStrict = VINF_SUCCESS;
13997	}
13998
13999	return rcStrict;
14000	}
14001	/** @} */
14002
14003
14004	/** @name VM-exit handlers.
14005	* @{
14006	*/
14007	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14008	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14009	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14010
14011	/**
14012	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14013	*/
14014	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14015	{
14016	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14018	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14019	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14020	return VINF_SUCCESS;
14021	return VINF_EM_RAW_INTERRUPT;
14022	}
14023
14024
14025	/**
14026	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14027	* VM-exit.
14028	*/
14029	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14030	{
14031	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14032	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14033
14034	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14035
14036	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14037	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14038	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14039
14040	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14041	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14042	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14043	NOREF(pVmcsInfo);
14044
14045	VBOXSTRICTRC rcStrict;
14046	switch (uExitIntType)
14047	{
14048	/*
14049	* Host physical NMIs:
14050	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14051	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14052	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14053	*
14054	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14055	* See Intel spec. 27.5.5 "Updating Non-Register State".
14056	*/
14057	case VMX_EXIT_INT_INFO_TYPE_NMI:
14058	{
14059	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14060	break;
14061	}
14062
14063	/*
14064	* Privileged software exceptions (#DB from ICEBP),
14065	* Software exceptions (#BP and #OF),
14066	* Hardware exceptions:
14067	* Process the required exceptions and resume guest execution if possible.
14068	*/
14069	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14070	Assert(uVector == X86_XCPT_DB);
14071	RT_FALL_THRU();
14072	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14073	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14074	RT_FALL_THRU();
14075	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14076	{
14077	NOREF(uVector);
14078	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14079	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14080	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14081	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14082
14083	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14084	break;
14085	}
14086
14087	default:
14088	{
14089	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14090	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14091	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14092	break;
14093	}
14094	}
14095
14096	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14097	return rcStrict;
14098	}
14099
14100
14101	/**
14102	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14103	*/
14104	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14105	{
14106	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14107
14108	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14109	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14110	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14111
14112	/* Evaluate and deliver pending events and resume guest execution. */
14113	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14114	return VINF_SUCCESS;
14115	}
14116
14117
14118	/**
14119	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14120	*/
14121	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14122	{
14123	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14124
14125	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14126	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14127	{
14128	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14129	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14130	}
14131
14132	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14133
14134	/*
14135	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14136	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14137	*/
14138	uint32_t fIntrState;
14139	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14140	AssertRC(rc);
14141	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14142	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14143	{
14144	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14145	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14146
14147	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14148	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14149	AssertRC(rc);
14150	}
14151
14152	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14153	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14154
14155	/* Evaluate and deliver pending events and resume guest execution. */
14156	return VINF_SUCCESS;
14157	}
14158
14159
14160	/**
14161	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14162	*/
14163	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14164	{
14165	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14166	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14167	}
14168
14169
14170	/**
14171	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14172	*/
14173	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14174	{
14175	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14176	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14177	}
14178
14179
14180	/**
14181	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14182	*/
14183	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14184	{
14185	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14186
14187	/*
14188	* Get the state we need and update the exit history entry.
14189	*/
14190	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14191	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14192
14193	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14194	AssertRCReturn(rc, rc);
14195
14196	VBOXSTRICTRC rcStrict;
14197	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14198	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14199	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14200	if (!pExitRec)
14201	{
14202	/*
14203	* Regular CPUID instruction execution.
14204	*/
14205	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
14206	if (rcStrict == VINF_SUCCESS)
14207	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14208	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14209	{
14210	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14211	rcStrict = VINF_SUCCESS;
14212	}
14213	}
14214	else
14215	{
14216	/*
14217	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14218	*/
14219	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14220	AssertRCReturn(rc2, rc2);
14221
14222	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14223	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14224
14225	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14226	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14227
14228	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14229	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14230	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14231	}
14232	return rcStrict;
14233	}
14234
14235
14236	/**
14237	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14238	*/
14239	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14240	{
14241	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14242
14243	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14244	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14245	AssertRCReturn(rc, rc);
14246
14247	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14248	return VINF_EM_RAW_EMULATE_INSTR;
14249
14250	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14251	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14252	}
14253
14254
14255	/**
14256	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14257	*/
14258	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14259	{
14260	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14261
14262	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14263	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14264	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14265	AssertRCReturn(rc, rc);
14266
14267	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
14268	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14269	{
14270	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14271	we must reset offsetting on VM-entry. See @bugref{6634}. */
14272	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14273	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14274	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14275	}
14276	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14277	{
14278	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14279	rcStrict = VINF_SUCCESS;
14280	}
14281	return rcStrict;
14282	}
14283
14284
14285	/**
14286	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14287	*/
14288	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14289	{
14290	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14291
14292	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14293	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14294	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14295	AssertRCReturn(rc, rc);
14296
14297	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
14298	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14299	{
14300	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14301	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14302	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14303	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14304	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14305	}
14306	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14307	{
14308	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14309	rcStrict = VINF_SUCCESS;
14310	}
14311	return rcStrict;
14312	}
14313
14314
14315	/**
14316	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14317	*/
14318	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14319	{
14320	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14321
14322	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14323	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14324	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14325	AssertRCReturn(rc, rc);
14326
14327	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14328	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14329	if (RT_LIKELY(rc == VINF_SUCCESS))
14330	{
14331	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14332	Assert(pVmxTransient->cbExitInstr == 2);
14333	}
14334	else
14335	{
14336	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14337	rc = VERR_EM_INTERPRETER;
14338	}
14339	return rc;
14340	}
14341
14342
14343	/**
14344	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14345	*/
14346	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14347	{
14348	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14349
14350	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14351	if (EMAreHypercallInstructionsEnabled(pVCpu))
14352	{
14353	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14354	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14355	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14356	AssertRCReturn(rc, rc);
14357
14358	/* Perform the hypercall. */
14359	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14360	if (rcStrict == VINF_SUCCESS)
14361	{
14362	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14363	AssertRCReturn(rc, rc);
14364	}
14365	else
14366	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14367	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14368	\|\| RT_FAILURE(rcStrict));
14369
14370	/* If the hypercall changes anything other than guest's general-purpose registers,
14371	we would need to reload the guest changed bits here before VM-entry. */
14372	}
14373	else
14374	Log4Func(("Hypercalls not enabled\n"));
14375
14376	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14377	if (RT_FAILURE(rcStrict))
14378	{
14379	hmR0VmxSetPendingXcptUD(pVCpu);
14380	rcStrict = VINF_SUCCESS;
14381	}
14382
14383	return rcStrict;
14384	}
14385
14386
14387	/**
14388	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14389	*/
14390	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14391	{
14392	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14393	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14394
14395	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14396	hmR0VmxReadExitQualVmcs(pVmxTransient);
14397	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14398	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14399	AssertRCReturn(rc, rc);
14400
14401	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
14402
14403	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14404	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14405	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14406	{
14407	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14408	rcStrict = VINF_SUCCESS;
14409	}
14410	else
14411	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14412	VBOXSTRICTRC_VAL(rcStrict)));
14413	return rcStrict;
14414	}
14415
14416
14417	/**
14418	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14419	*/
14420	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14421	{
14422	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14423
14424	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14425	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14426	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14427	AssertRCReturn(rc, rc);
14428
14429	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
14430	if (rcStrict == VINF_SUCCESS)
14431	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14432	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14433	{
14434	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14435	rcStrict = VINF_SUCCESS;
14436	}
14437
14438	return rcStrict;
14439	}
14440
14441
14442	/**
14443	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14444	*/
14445	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14446	{
14447	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14448
14449	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14450	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14451	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14452	AssertRCReturn(rc, rc);
14453
14454	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
14455	if (RT_SUCCESS(rcStrict))
14456	{
14457	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14458	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14459	rcStrict = VINF_SUCCESS;
14460	}
14461
14462	return rcStrict;
14463	}
14464
14465
14466	/**
14467	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14468	* VM-exit.
14469	*/
14470	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14471	{
14472	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14473	return VINF_EM_RESET;
14474	}
14475
14476
14477	/**
14478	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14479	*/
14480	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14481	{
14482	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14483
14484	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14485	AssertRCReturn(rc, rc);
14486
14487	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14488	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14489	rc = VINF_SUCCESS;
14490	else
14491	rc = VINF_EM_HALT;
14492
14493	if (rc != VINF_SUCCESS)
14494	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14495	return rc;
14496	}
14497
14498
14499	/**
14500	* VM-exit handler for instructions that result in a \#UD exception delivered to
14501	* the guest.
14502	*/
14503	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14504	{
14505	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14506	hmR0VmxSetPendingXcptUD(pVCpu);
14507	return VINF_SUCCESS;
14508	}
14509
14510
14511	/**
14512	* VM-exit handler for expiry of the VMX-preemption timer.
14513	*/
14514	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14515	{
14516	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14517
14518	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14519	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14520
14521	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14522	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14523	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14524	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14525	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14526	}
14527
14528
14529	/**
14530	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14531	*/
14532	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14533	{
14534	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14535
14536	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14537	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14538	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14539	AssertRCReturn(rc, rc);
14540
14541	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
14542	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14543	: HM_CHANGED_RAISED_XCPT_MASK);
14544
14545	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14546	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14547
14548	return rcStrict;
14549	}
14550
14551
14552	/**
14553	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14554	*/
14555	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14556	{
14557	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14558
14559	#if 1
14560	/** @todo Use VM-exit instruction information. */
14561	return VERR_EM_INTERPRETER;
14562	#else
14563	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14564	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14565	hmR0VmxReadExitQualVmcs(pVmxTransient);
14566	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
14567	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14568	AssertRCReturn(rc, rc);
14569
14570	/* Paranoia. Ensure this has a memory operand. */
14571	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
14572
14573	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
14574	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14575	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
14576	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
14577
14578	RTGCPTR GCPtrDesc;
14579	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
14580
14581	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
14582	GCPtrDesc, uType);
14583	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14584	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14585	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14586	{
14587	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14588	rcStrict = VINF_SUCCESS;
14589	}
14590	return rcStrict;
14591	#endif
14592	}
14593
14594
14595	/**
14596	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14597	* VM-exit.
14598	*/
14599	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14600	{
14601	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14602	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14603	AssertRCReturn(rc, rc);
14604
14605	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14606	if (RT_FAILURE(rc))
14607	return rc;
14608
14609	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14610	NOREF(uInvalidReason);
14611
14612	#ifdef VBOX_STRICT
14613	uint32_t fIntrState;
14614	uint64_t u64Val;
14615	hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14616	hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14617	hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14618
14619	Log4(("uInvalidReason %u\n", uInvalidReason));
14620	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14621	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14622	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14623
14624	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
14625	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14626	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
14627	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
14628	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
14629	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
14630	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
14631	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14632	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
14633	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
14634	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
14635	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14636	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14637	{
14638	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14639	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14640	}
14641	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
14642	#endif
14643
14644	return VERR_VMX_INVALID_GUEST_STATE;
14645	}
14646
14647	/**
14648	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14649	*/
14650	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14651	{
14652	/*
14653	* Cumulative notes of all recognized but unexpected VM-exits.
14654	*
14655	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
14656	* nested-paging is used.
14657	*
14658	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14659	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14660	* this function (and thereby stop VM execution) for handling such instructions.
14661	*
14662	*
14663	* VMX_EXIT_INIT_SIGNAL:
14664	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14665	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14666	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14667	*
14668	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14669	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14670	* See Intel spec. "23.8 Restrictions on VMX operation".
14671	*
14672	* VMX_EXIT_SIPI:
14673	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14674	* activity state is used. We don't make use of it as our guests don't have direct
14675	* access to the host local APIC.
14676	*
14677	* See Intel spec. 25.3 "Other Causes of VM-exits".
14678	*
14679	* VMX_EXIT_IO_SMI:
14680	* VMX_EXIT_SMI:
14681	* This can only happen if we support dual-monitor treatment of SMI, which can be
14682	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14683	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14684	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14685	*
14686	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14687	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14688	*
14689	* VMX_EXIT_ERR_MSR_LOAD:
14690	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14691	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14692	* execution.
14693	*
14694	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14695	*
14696	* VMX_EXIT_ERR_MACHINE_CHECK:
14697	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14698	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14699	* #MC exception abort class exception is raised. We thus cannot assume a
14700	* reasonable chance of continuing any sort of execution and we bail.
14701	*
14702	* See Intel spec. 15.1 "Machine-check Architecture".
14703	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14704	*
14705	* VMX_EXIT_PML_FULL:
14706	* VMX_EXIT_VIRTUALIZED_EOI:
14707	* VMX_EXIT_APIC_WRITE:
14708	* We do not currently support any of these features and thus they are all unexpected
14709	* VM-exits.
14710	*
14711	* VMX_EXIT_GDTR_IDTR_ACCESS:
14712	* VMX_EXIT_LDTR_TR_ACCESS:
14713	* VMX_EXIT_RDRAND:
14714	* VMX_EXIT_RSM:
14715	* VMX_EXIT_VMFUNC:
14716	* VMX_EXIT_ENCLS:
14717	* VMX_EXIT_RDSEED:
14718	* VMX_EXIT_XSAVES:
14719	* VMX_EXIT_XRSTORS:
14720	* VMX_EXIT_UMWAIT:
14721	* VMX_EXIT_TPAUSE:
14722	* These VM-exits are -not- caused unconditionally by execution of the corresponding
14723	* instruction. Any VM-exit for these instructions indicate a hardware problem,
14724	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14725	*
14726	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14727	*/
14728	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14729	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14730	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14731	}
14732
14733
14734	/**
14735	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14736	*/
14737	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14738	{
14739	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14740
14741	/** @todo Optimize this: We currently drag in the whole MSR state
14742	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14743	* MSRs required. That would require changes to IEM and possibly CPUM too.
14744	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14745	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14746	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14747	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14748	switch (idMsr)
14749	{
14750	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14751	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14752	}
14753
14754	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14755	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14756	AssertRCReturn(rc, rc);
14757
14758	Log4Func(("ecx=%#RX32\n", idMsr));
14759
14760	#ifdef VBOX_STRICT
14761	Assert(!pVmxTransient->fIsNestedGuest);
14762	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
14763	{
14764	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14765	&& idMsr != MSR_K6_EFER)
14766	{
14767	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14768	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14769	}
14770	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14771	{
14772	Assert(pVmcsInfo->pvMsrBitmap);
14773	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14774	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14775	{
14776	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14777	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14778	}
14779	}
14780	}
14781	#endif
14782
14783	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
14784	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14785	if (rcStrict == VINF_SUCCESS)
14786	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14787	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14788	{
14789	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14790	rcStrict = VINF_SUCCESS;
14791	}
14792	else
14793	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14794
14795	return rcStrict;
14796	}
14797
14798
14799	/**
14800	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14801	*/
14802	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14803	{
14804	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14805
14806	/** @todo Optimize this: We currently drag in the whole MSR state
14807	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14808	* MSRs required. That would require changes to IEM and possibly CPUM too.
14809	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14810	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14811	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14812
14813	/*
14814	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14815	* Although we don't need to fetch the base as it will be overwritten shortly, while
14816	* loading guest-state we would also load the entire segment register including limit
14817	* and attributes and thus we need to load them here.
14818	*/
14819	switch (idMsr)
14820	{
14821	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14822	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14823	}
14824
14825	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14826	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14827	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14828	AssertRCReturn(rc, rc);
14829
14830	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14831
14832	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
14833	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14834
14835	if (rcStrict == VINF_SUCCESS)
14836	{
14837	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14838
14839	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14840	if ( idMsr == MSR_IA32_APICBASE
14841	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14842	&& idMsr <= MSR_IA32_X2APIC_END))
14843	{
14844	/*
14845	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14846	* When full APIC register virtualization is implemented we'll have to make
14847	* sure APIC state is saved from the VMCS before IEM changes it.
14848	*/
14849	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14850	}
14851	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14852	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14853	else if (idMsr == MSR_K6_EFER)
14854	{
14855	/*
14856	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14857	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14858	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14859	*/
14860	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14861	}
14862
14863	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
14864	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14865	{
14866	switch (idMsr)
14867	{
14868	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14869	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14870	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14871	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14872	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14873	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14874	default:
14875	{
14876	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14877	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14878	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14879	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14880	break;
14881	}
14882	}
14883	}
14884	#ifdef VBOX_STRICT
14885	else
14886	{
14887	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14888	switch (idMsr)
14889	{
14890	case MSR_IA32_SYSENTER_CS:
14891	case MSR_IA32_SYSENTER_EIP:
14892	case MSR_IA32_SYSENTER_ESP:
14893	case MSR_K8_FS_BASE:
14894	case MSR_K8_GS_BASE:
14895	{
14896	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14897	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14898	}
14899
14900	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14901	default:
14902	{
14903	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14904	{
14905	/* EFER MSR writes are always intercepted. */
14906	if (idMsr != MSR_K6_EFER)
14907	{
14908	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14909	idMsr));
14910	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14911	}
14912	}
14913
14914	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14915	{
14916	Assert(pVmcsInfo->pvMsrBitmap);
14917	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14918	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14919	{
14920	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14921	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14922	}
14923	}
14924	break;
14925	}
14926	}
14927	}
14928	#endif /* VBOX_STRICT */
14929	}
14930	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14931	{
14932	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14933	rcStrict = VINF_SUCCESS;
14934	}
14935	else
14936	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14937
14938	return rcStrict;
14939	}
14940
14941
14942	/**
14943	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14944	*/
14945	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14946	{
14947	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14948
14949	/** @todo The guest has likely hit a contended spinlock. We might want to
14950	* poke a schedule different guest VCPU. */
14951	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14952	if (RT_SUCCESS(rc))
14953	return VINF_EM_RAW_INTERRUPT;
14954
14955	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14956	return rc;
14957	}
14958
14959
14960	/**
14961	* VM-exit handler for when the TPR value is lowered below the specified
14962	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14963	*/
14964	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14965	{
14966	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14967	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14968
14969	/*
14970	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14971	* We'll re-evaluate pending interrupts and inject them before the next VM
14972	* entry so we can just continue execution here.
14973	*/
14974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14975	return VINF_SUCCESS;
14976	}
14977
14978
14979	/**
14980	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14981	* VM-exit.
14982	*
14983	* @retval VINF_SUCCESS when guest execution can continue.
14984	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14985	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
14986	* incompatible guest state for VMX execution (real-on-v86 case).
14987	*/
14988	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14989	{
14990	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14991	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14992
14993	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14994	hmR0VmxReadExitQualVmcs(pVmxTransient);
14995	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14996
14997	VBOXSTRICTRC rcStrict;
14998	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14999	uint64_t const uExitQual = pVmxTransient->uExitQual;
15000	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15001	switch (uAccessType)
15002	{
15003	/*
15004	* MOV to CRx.
15005	*/
15006	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15007	{
15008	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15009	AssertRCReturn(rc, rc);
15010
15011	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15012	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15013	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15014	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15015
15016	/*
15017	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15018	* - When nested paging isn't used.
15019	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15020	* - We are executing in the VM debug loop.
15021	*/
15022	Assert( iCrReg != 3
15023	\|\| !pVM->hm.s.fNestedPaging
15024	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15025	\|\| pVCpu->hm.s.fUsingDebugLoop);
15026
15027	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15028	Assert( iCrReg != 8
15029	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15030
15031	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15032	AssertMsg( rcStrict == VINF_SUCCESS
15033	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15034
15035	/*
15036	* This is a kludge for handling switches back to real mode when we try to use
15037	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15038	* deal with special selector values, so we have to return to ring-3 and run
15039	* there till the selector values are V86 mode compatible.
15040	*
15041	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15042	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15043	* this function.
15044	*/
15045	if ( iCrReg == 0
15046	&& rcStrict == VINF_SUCCESS
15047	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15048	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15049	&& (uOldCr0 & X86_CR0_PE)
15050	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15051	{
15052	/** @todo Check selectors rather than returning all the time. */
15053	Assert(!pVmxTransient->fIsNestedGuest);
15054	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15055	rcStrict = VINF_EM_RESCHEDULE_REM;
15056	}
15057	break;
15058	}
15059
15060	/*
15061	* MOV from CRx.
15062	*/
15063	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15064	{
15065	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15066	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15067
15068	/*
15069	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15070	* - When nested paging isn't used.
15071	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15072	* - We are executing in the VM debug loop.
15073	*/
15074	Assert( iCrReg != 3
15075	\|\| !pVM->hm.s.fNestedPaging
15076	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15077	\|\| pVCpu->hm.s.fUsingDebugLoop);
15078
15079	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15080	Assert( iCrReg != 8
15081	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15082
15083	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15084	break;
15085	}
15086
15087	/*
15088	* CLTS (Clear Task-Switch Flag in CR0).
15089	*/
15090	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15091	{
15092	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
15093	break;
15094	}
15095
15096	/*
15097	* LMSW (Load Machine-Status Word into CR0).
15098	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15099	*/
15100	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15101	{
15102	RTGCPTR GCPtrEffDst;
15103	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
15104	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15105	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15106	if (fMemOperand)
15107	{
15108	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
15109	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15110	}
15111	else
15112	GCPtrEffDst = NIL_RTGCPTR;
15113	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15114	break;
15115	}
15116
15117	default:
15118	{
15119	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15120	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15121	}
15122	}
15123
15124	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15125	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15126	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15127
15128	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15129	NOREF(pVM);
15130	return rcStrict;
15131	}
15132
15133
15134	/**
15135	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15136	* VM-exit.
15137	*/
15138	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15139	{
15140	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15141	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15142
15143	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15144	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15145	hmR0VmxReadExitQualVmcs(pVmxTransient);
15146	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15147	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15148	\| CPUMCTX_EXTRN_EFER);
15149	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15150	AssertRCReturn(rc, rc);
15151
15152	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15153	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15154	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15155	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15156	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15157	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15158	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15159	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15160
15161	/*
15162	* Update exit history to see if this exit can be optimized.
15163	*/
15164	VBOXSTRICTRC rcStrict;
15165	PCEMEXITREC pExitRec = NULL;
15166	if ( !fGstStepping
15167	&& !fDbgStepping)
15168	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15169	!fIOString
15170	? !fIOWrite
15171	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15172	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15173	: !fIOWrite
15174	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15175	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15176	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15177	if (!pExitRec)
15178	{
15179	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15180	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15181
15182	uint32_t const cbValue = s_aIOSizes[uIOSize];
15183	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
15184	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15185	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15186	if (fIOString)
15187	{
15188	/*
15189	* INS/OUTS - I/O String instruction.
15190	*
15191	* Use instruction-information if available, otherwise fall back on
15192	* interpreting the instruction.
15193	*/
15194	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15195	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15196	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15197	if (fInsOutsInfo)
15198	{
15199	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15200	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15201	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15202	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15203	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15204	if (fIOWrite)
15205	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15206	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15207	else
15208	{
15209	/*
15210	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15211	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15212	* See Intel Instruction spec. for "INS".
15213	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15214	*/
15215	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15216	}
15217	}
15218	else
15219	rcStrict = IEMExecOne(pVCpu);
15220
15221	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15222	fUpdateRipAlready = true;
15223	}
15224	else
15225	{
15226	/*
15227	* IN/OUT - I/O instruction.
15228	*/
15229	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15230	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15231	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15232	if (fIOWrite)
15233	{
15234	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15235	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15236	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15237	&& !pCtx->eflags.Bits.u1TF)
15238	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15239	}
15240	else
15241	{
15242	uint32_t u32Result = 0;
15243	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15244	if (IOM_SUCCESS(rcStrict))
15245	{
15246	/* Save result of I/O IN instr. in AL/AX/EAX. */
15247	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15248	}
15249	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15250	&& !pCtx->eflags.Bits.u1TF)
15251	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15252	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15253	}
15254	}
15255
15256	if (IOM_SUCCESS(rcStrict))
15257	{
15258	if (!fUpdateRipAlready)
15259	{
15260	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15261	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15262	}
15263
15264	/*
15265	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15266	* while booting Fedora 17 64-bit guest.
15267	*
15268	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15269	*/
15270	if (fIOString)
15271	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15272
15273	/*
15274	* If any I/O breakpoints are armed, we need to check if one triggered
15275	* and take appropriate action.
15276	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15277	*/
15278	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15279	AssertRCReturn(rc, rc);
15280
15281	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15282	* execution engines about whether hyper BPs and such are pending. */
15283	uint32_t const uDr7 = pCtx->dr[7];
15284	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15285	&& X86_DR7_ANY_RW_IO(uDr7)
15286	&& (pCtx->cr4 & X86_CR4_DE))
15287	\|\| DBGFBpIsHwIoArmed(pVM)))
15288	{
15289	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15290
15291	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15292	VMMRZCallRing3Disable(pVCpu);
15293	HM_DISABLE_PREEMPT(pVCpu);
15294
15295	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15296
15297	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15298	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15299	{
15300	/* Raise #DB. */
15301	if (fIsGuestDbgActive)
15302	ASMSetDR6(pCtx->dr[6]);
15303	if (pCtx->dr[7] != uDr7)
15304	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15305
15306	hmR0VmxSetPendingXcptDB(pVCpu);
15307	}
15308	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15309	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15310	else if ( rcStrict2 != VINF_SUCCESS
15311	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15312	rcStrict = rcStrict2;
15313	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15314
15315	HM_RESTORE_PREEMPT();
15316	VMMRZCallRing3Enable(pVCpu);
15317	}
15318	}
15319
15320	#ifdef VBOX_STRICT
15321	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15322	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15323	Assert(!fIOWrite);
15324	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15325	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15326	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15327	Assert(fIOWrite);
15328	else
15329	{
15330	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15331	* statuses, that the VMM device and some others may return. See
15332	* IOM_SUCCESS() for guidance. */
15333	AssertMsg( RT_FAILURE(rcStrict)
15334	\|\| rcStrict == VINF_SUCCESS
15335	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15336	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15337	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15338	\|\| rcStrict == VINF_EM_RAW_TO_R3
15339	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15340	# endif
15341	}
15342	#endif
15343	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15344	}
15345	else
15346	{
15347	/*
15348	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15349	*/
15350	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15351	AssertRCReturn(rc2, rc2);
15352	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15353	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15354	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15355	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15356	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15357	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15358
15359	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15360	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15361
15362	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15363	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15364	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15365	}
15366	return rcStrict;
15367	}
15368
15369
15370	/**
15371	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15372	* VM-exit.
15373	*/
15374	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15375	{
15376	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15377
15378	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15379	hmR0VmxReadExitQualVmcs(pVmxTransient);
15380	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15381	{
15382	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15383	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15384	{
15385	uint32_t uErrCode;
15386	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15387	{
15388	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15389	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15390	}
15391	else
15392	uErrCode = 0;
15393
15394	RTGCUINTPTR GCPtrFaultAddress;
15395	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15396	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15397	else
15398	GCPtrFaultAddress = 0;
15399
15400	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15401
15402	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15403	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
15404
15405	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15406	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15407	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15408	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15409	}
15410	}
15411
15412	/* Fall back to the interpreter to emulate the task-switch. */
15413	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15414	return VERR_EM_INTERPRETER;
15415	}
15416
15417
15418	/**
15419	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15420	*/
15421	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15422	{
15423	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15424
15425	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15426	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15427	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15428	AssertRC(rc);
15429	return VINF_EM_DBG_STEPPED;
15430	}
15431
15432
15433	/**
15434	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15435	*/
15436	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15437	{
15438	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15439	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15440
15441	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15442	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15443	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15444	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15445	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15446
15447	/*
15448	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15449	*/
15450	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15451	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15452	{
15453	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15454	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15455	{
15456	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15457	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15458	}
15459	}
15460	else
15461	{
15462	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15463	return rcStrict;
15464	}
15465
15466	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15467	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15468	hmR0VmxReadExitQualVmcs(pVmxTransient);
15469	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15470	AssertRCReturn(rc, rc);
15471
15472	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15473	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15474	switch (uAccessType)
15475	{
15476	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15477	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15478	{
15479	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15480	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15481	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15482
15483	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15484	GCPhys &= PAGE_BASE_GC_MASK;
15485	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15486	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15487	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15488
15489	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15490	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15491	rcStrict = IOMMMIOPhysHandler(pVM, pVCpu,
15492	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15493	CPUMCTX2CORE(pCtx), GCPhys);
15494	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15495	if ( rcStrict == VINF_SUCCESS
15496	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15497	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15498	{
15499	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15500	\| HM_CHANGED_GUEST_APIC_TPR);
15501	rcStrict = VINF_SUCCESS;
15502	}
15503	break;
15504	}
15505
15506	default:
15507	{
15508	Log4Func(("uAccessType=%#x\n", uAccessType));
15509	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15510	break;
15511	}
15512	}
15513
15514	if (rcStrict != VINF_SUCCESS)
15515	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15516	return rcStrict;
15517	}
15518
15519
15520	/**
15521	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15522	* VM-exit.
15523	*/
15524	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15525	{
15526	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15527	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15528
15529	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15530	if (!pVmxTransient->fIsNestedGuest)
15531	{
15532	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15533	if (pVmxTransient->fWasGuestDebugStateActive)
15534	{
15535	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15536	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15537	}
15538
15539	if ( !pVCpu->hm.s.fSingleInstruction
15540	&& !pVmxTransient->fWasHyperDebugStateActive)
15541	{
15542	Assert(!DBGFIsStepping(pVCpu));
15543	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15544
15545	/* Don't intercept MOV DRx any more. */
15546	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15547	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15548	AssertRC(rc);
15549
15550	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15551	VMMRZCallRing3Disable(pVCpu);
15552	HM_DISABLE_PREEMPT(pVCpu);
15553
15554	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15555	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15556	Assert(CPUMIsGuestDebugStateActive(pVCpu));
15557
15558	HM_RESTORE_PREEMPT();
15559	VMMRZCallRing3Enable(pVCpu);
15560
15561	#ifdef VBOX_WITH_STATISTICS
15562	hmR0VmxReadExitQualVmcs(pVmxTransient);
15563	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15564	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15565	else
15566	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15567	#endif
15568	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15569	return VINF_SUCCESS;
15570	}
15571	}
15572
15573	/*
15574	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15575	* The EFER MSR is always up-to-date.
15576	* Update the segment registers and DR7 from the CPU.
15577	*/
15578	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15579	hmR0VmxReadExitQualVmcs(pVmxTransient);
15580	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15581	AssertRCReturn(rc, rc);
15582	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15583
15584	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15585	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15586	{
15587	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15588	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15589	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15590	if (RT_SUCCESS(rc))
15591	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15593	}
15594	else
15595	{
15596	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15597	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15598	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15599	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15600	}
15601
15602	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15603	if (RT_SUCCESS(rc))
15604	{
15605	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15606	AssertRCReturn(rc2, rc2);
15607	return VINF_SUCCESS;
15608	}
15609	return rc;
15610	}
15611
15612
15613	/**
15614	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15615	* Conditional VM-exit.
15616	*/
15617	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15618	{
15619	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15620	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15621
15622	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15623	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15624	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15625	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15626	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15627
15628	/*
15629	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15630	*/
15631	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15632	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15633	{
15634	/*
15635	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15636	* instruction emulation to inject the original event. Otherwise, injecting the original event
15637	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
15638	*/
15639	if (!pVCpu->hm.s.Event.fPending)
15640	{ /* likely */ }
15641	else
15642	{
15643	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15644	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15645	/** @todo NSTVMX: Think about how this should be handled. */
15646	if (pVmxTransient->fIsNestedGuest)
15647	return VERR_VMX_IPE_3;
15648	#endif
15649	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15650	}
15651	}
15652	else
15653	{
15654	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15655	return rcStrict;
15656	}
15657
15658	/*
15659	* Get sufficient state and update the exit history entry.
15660	*/
15661	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15662	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15663	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15664	AssertRCReturn(rc, rc);
15665
15666	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15667	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15668	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15669	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15670	if (!pExitRec)
15671	{
15672	/*
15673	* If we succeed, resume guest execution.
15674	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15675	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15676	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15677	* weird case. See @bugref{6043}.
15678	*/
15679	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15680	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15681	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15682	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15683	if ( rcStrict == VINF_SUCCESS
15684	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15685	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15686	{
15687	/* Successfully handled MMIO operation. */
15688	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15689	\| HM_CHANGED_GUEST_APIC_TPR);
15690	rcStrict = VINF_SUCCESS;
15691	}
15692	}
15693	else
15694	{
15695	/*
15696	* Frequent exit or something needing probing. Call EMHistoryExec.
15697	*/
15698	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15699	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15700
15701	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15702	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15703
15704	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15705	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15706	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15707	}
15708	return rcStrict;
15709	}
15710
15711
15712	/**
15713	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15714	* VM-exit.
15715	*/
15716	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15717	{
15718	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15719	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15720
15721	hmR0VmxReadExitQualVmcs(pVmxTransient);
15722	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15723	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15724	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15725	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15726	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15727
15728	/*
15729	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15730	*/
15731	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15732	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15733	{
15734	/*
15735	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
15736	* we shall resolve the nested #PF and re-inject the original event.
15737	*/
15738	if (pVCpu->hm.s.Event.fPending)
15739	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
15740	}
15741	else
15742	{
15743	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15744	return rcStrict;
15745	}
15746
15747	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15748	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15749	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15750	AssertRCReturn(rc, rc);
15751
15752	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15753	uint64_t const uExitQual = pVmxTransient->uExitQual;
15754	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
15755
15756	RTGCUINT uErrorCode = 0;
15757	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15758	uErrorCode \|= X86_TRAP_PF_ID;
15759	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15760	uErrorCode \|= X86_TRAP_PF_RW;
15761	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15762	uErrorCode \|= X86_TRAP_PF_P;
15763
15764	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15765	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15766	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
15767
15768	/*
15769	* Handle the pagefault trap for the nested shadow table.
15770	*/
15771	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15772	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15773	TRPMResetTrap(pVCpu);
15774
15775	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15776	if ( rcStrict == VINF_SUCCESS
15777	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15778	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15779	{
15780	/* Successfully synced our nested page tables. */
15781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15782	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15783	return VINF_SUCCESS;
15784	}
15785
15786	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15787	return rcStrict;
15788	}
15789
15790
15791	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15792	/**
15793	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15794	*/
15795	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15796	{
15797	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15798
15799	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15800	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15801	hmR0VmxReadExitQualVmcs(pVmxTransient);
15802	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15803	\| CPUMCTX_EXTRN_HWVIRT
15804	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15805	AssertRCReturn(rc, rc);
15806
15807	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15808
15809	VMXVEXITINFO ExitInfo;
15810	RT_ZERO(ExitInfo);
15811	ExitInfo.uReason = pVmxTransient->uExitReason;
15812	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15813	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15814	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15815	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15816
15817	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15818	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15819	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15820	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15821	{
15822	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15823	rcStrict = VINF_SUCCESS;
15824	}
15825	return rcStrict;
15826	}
15827
15828
15829	/**
15830	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15831	*/
15832	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15833	{
15834	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15835
15836	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15837	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15838	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15839	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15840	AssertRCReturn(rc, rc);
15841
15842	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15843
15844	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
15845	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
15846	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
15847	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15848	{
15849	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15850	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
15851	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15852	}
15853	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15854	return rcStrict;
15855	}
15856
15857
15858	/**
15859	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15860	*/
15861	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15862	{
15863	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15864
15865	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15866	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15867	hmR0VmxReadExitQualVmcs(pVmxTransient);
15868	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15869	\| CPUMCTX_EXTRN_HWVIRT
15870	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15871	AssertRCReturn(rc, rc);
15872
15873	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15874
15875	VMXVEXITINFO ExitInfo;
15876	RT_ZERO(ExitInfo);
15877	ExitInfo.uReason = pVmxTransient->uExitReason;
15878	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15879	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15880	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15881	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15882
15883	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15884	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15885	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15886	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15887	{
15888	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15889	rcStrict = VINF_SUCCESS;
15890	}
15891	return rcStrict;
15892	}
15893
15894
15895	/**
15896	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15897	*/
15898	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15899	{
15900	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15901
15902	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15903	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15904	hmR0VmxReadExitQualVmcs(pVmxTransient);
15905	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15906	\| CPUMCTX_EXTRN_HWVIRT
15907	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15908	AssertRCReturn(rc, rc);
15909
15910	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15911
15912	VMXVEXITINFO ExitInfo;
15913	RT_ZERO(ExitInfo);
15914	ExitInfo.uReason = pVmxTransient->uExitReason;
15915	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15916	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15917	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15918	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15919
15920	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15921	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15922	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15923	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15924	{
15925	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15926	rcStrict = VINF_SUCCESS;
15927	}
15928	return rcStrict;
15929	}
15930
15931
15932	/**
15933	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
15934	*/
15935	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15936	{
15937	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15938
15939	/*
15940	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
15941	* thus might not need to import the shadow VMCS state, it's safer just in case
15942	* code elsewhere dares look at unsynced VMCS fields.
15943	*/
15944	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15945	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15946	hmR0VmxReadExitQualVmcs(pVmxTransient);
15947	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15948	\| CPUMCTX_EXTRN_HWVIRT
15949	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15950	AssertRCReturn(rc, rc);
15951
15952	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15953
15954	VMXVEXITINFO ExitInfo;
15955	RT_ZERO(ExitInfo);
15956	ExitInfo.uReason = pVmxTransient->uExitReason;
15957	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15958	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15959	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15960	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15961	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15962
15963	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15964	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15965	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15966	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15967	{
15968	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15969	rcStrict = VINF_SUCCESS;
15970	}
15971	return rcStrict;
15972	}
15973
15974
15975	/**
15976	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15977	*/
15978	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15979	{
15980	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15981
15982	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15983	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15984	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15985	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15986	AssertRCReturn(rc, rc);
15987
15988	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15989
15990	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
15991	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
15992	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
15993	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15994	{
15995	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15996	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
15997	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15998	}
15999	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16000	return rcStrict;
16001	}
16002
16003
16004	/**
16005	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16006	*/
16007	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16008	{
16009	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16010
16011	/*
16012	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16013	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16014	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16015	*/
16016	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16017	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16018	hmR0VmxReadExitQualVmcs(pVmxTransient);
16019	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16020	\| CPUMCTX_EXTRN_HWVIRT
16021	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16022	AssertRCReturn(rc, rc);
16023
16024	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16025
16026	VMXVEXITINFO ExitInfo;
16027	RT_ZERO(ExitInfo);
16028	ExitInfo.uReason = pVmxTransient->uExitReason;
16029	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16030	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16031	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16032	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16033	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16034
16035	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16036	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16037	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16038	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16039	{
16040	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16041	rcStrict = VINF_SUCCESS;
16042	}
16043	return rcStrict;
16044	}
16045
16046
16047	/**
16048	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16049	*/
16050	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16051	{
16052	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16053
16054	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16055	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16056	\| CPUMCTX_EXTRN_HWVIRT
16057	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16058	AssertRCReturn(rc, rc);
16059
16060	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16061
16062	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
16063	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16064	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16065	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16066	{
16067	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16068	rcStrict = VINF_SUCCESS;
16069	}
16070	return rcStrict;
16071	}
16072
16073
16074	/**
16075	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16076	*/
16077	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16078	{
16079	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16080
16081	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16082	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16083	hmR0VmxReadExitQualVmcs(pVmxTransient);
16084	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16085	\| CPUMCTX_EXTRN_HWVIRT
16086	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16087	AssertRCReturn(rc, rc);
16088
16089	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16090
16091	VMXVEXITINFO ExitInfo;
16092	RT_ZERO(ExitInfo);
16093	ExitInfo.uReason = pVmxTransient->uExitReason;
16094	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16095	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16096	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16097	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16098
16099	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16100	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16101	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16102	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16103	{
16104	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16105	rcStrict = VINF_SUCCESS;
16106	}
16107	return rcStrict;
16108	}
16109
16110
16111	/**
16112	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16113	*/
16114	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16115	{
16116	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16117
16118	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16119	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16120	hmR0VmxReadExitQualVmcs(pVmxTransient);
16121	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16122	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16123	AssertRCReturn(rc, rc);
16124
16125	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16126
16127	VMXVEXITINFO ExitInfo;
16128	RT_ZERO(ExitInfo);
16129	ExitInfo.uReason = pVmxTransient->uExitReason;
16130	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16131	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16132	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16133	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16134
16135	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16136	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16137	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16138	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16139	{
16140	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16141	rcStrict = VINF_SUCCESS;
16142	}
16143	return rcStrict;
16144	}
16145	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16146	/** @} */
16147
16148
16149	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16150	/** @name Nested-guest VM-exit handlers.
16151	* @{
16152	*/
16153	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16154	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16155	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16156
16157	/**
16158	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16159	* Conditional VM-exit.
16160	*/
16161	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16162	{
16163	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16164
16165	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16166
16167	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16168	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16169	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16170
16171	switch (uExitIntType)
16172	{
16173	/*
16174	* Physical NMIs:
16175	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16176	*/
16177	case VMX_EXIT_INT_INFO_TYPE_NMI:
16178	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16179
16180	/*
16181	* Hardware exceptions,
16182	* Software exceptions,
16183	* Privileged software exceptions:
16184	* Figure out if the exception must be delivered to the guest or the nested-guest.
16185	*/
16186	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16187	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16188	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16189	{
16190	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16191	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16192	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16193	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16194
16195	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16196	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16197	pVmxTransient->uExitIntErrorCode);
16198	if (fIntercept)
16199	{
16200	/* Exit qualification is required for debug and page-fault exceptions. */
16201	hmR0VmxReadExitQualVmcs(pVmxTransient);
16202
16203	/*
16204	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16205	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16206	* length. However, if delivery of a software interrupt, software exception or privileged
16207	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16208	*/
16209	VMXVEXITINFO ExitInfo;
16210	RT_ZERO(ExitInfo);
16211	ExitInfo.uReason = pVmxTransient->uExitReason;
16212	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16213	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16214
16215	VMXVEXITEVENTINFO ExitEventInfo;
16216	RT_ZERO(ExitEventInfo);
16217	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16218	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16219	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16220	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16221
16222	#ifdef DEBUG_ramshankar
16223	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16224	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
16225	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16226	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16227	{
16228	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
16229	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
16230	}
16231	#endif
16232	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16233	}
16234
16235	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16236	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16237	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16238	}
16239
16240	/*
16241	* Software interrupts:
16242	* VM-exits cannot be caused by software interrupts.
16243	*
16244	* External interrupts:
16245	* This should only happen when "acknowledge external interrupts on VM-exit"
16246	* control is set. However, we never set this when executing a guest or
16247	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16248	* the guest.
16249	*/
16250	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16251	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16252	default:
16253	{
16254	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16255	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16256	}
16257	}
16258	}
16259
16260
16261	/**
16262	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16263	* Unconditional VM-exit.
16264	*/
16265	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16266	{
16267	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16268	return IEMExecVmxVmexitTripleFault(pVCpu);
16269	}
16270
16271
16272	/**
16273	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16274	*/
16275	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16276	{
16277	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16278
16279	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16280	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16281	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16282	}
16283
16284
16285	/**
16286	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16287	*/
16288	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16289	{
16290	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16291
16292	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16293	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16294	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16295	}
16296
16297
16298	/**
16299	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16300	* Unconditional VM-exit.
16301	*/
16302	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16303	{
16304	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16305
16306	hmR0VmxReadExitQualVmcs(pVmxTransient);
16307	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16308	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16309	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16310
16311	VMXVEXITINFO ExitInfo;
16312	RT_ZERO(ExitInfo);
16313	ExitInfo.uReason = pVmxTransient->uExitReason;
16314	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16315	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16316
16317	VMXVEXITEVENTINFO ExitEventInfo;
16318	RT_ZERO(ExitEventInfo);
16319	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16320	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16321	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16322	}
16323
16324
16325	/**
16326	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16327	*/
16328	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16329	{
16330	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16331
16332	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16333	{
16334	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16335	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16336	}
16337	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16338	}
16339
16340
16341	/**
16342	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16343	*/
16344	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16345	{
16346	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16347
16348	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16349	{
16350	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16351	hmR0VmxReadExitQualVmcs(pVmxTransient);
16352
16353	VMXVEXITINFO ExitInfo;
16354	RT_ZERO(ExitInfo);
16355	ExitInfo.uReason = pVmxTransient->uExitReason;
16356	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16357	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16358	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16359	}
16360	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16361	}
16362
16363
16364	/**
16365	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16366	*/
16367	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16368	{
16369	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16370
16371	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16372	{
16373	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16374	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16375	}
16376	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16377	}
16378
16379
16380	/**
16381	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16382	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16383	*/
16384	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16385	{
16386	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16387
16388	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16389	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16390
16391	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16392
16393	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16394	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16395	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16396
16397	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16398	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16399	u64VmcsField &= UINT64_C(0xffffffff);
16400
16401	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16402	{
16403	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16404	hmR0VmxReadExitQualVmcs(pVmxTransient);
16405
16406	VMXVEXITINFO ExitInfo;
16407	RT_ZERO(ExitInfo);
16408	ExitInfo.uReason = pVmxTransient->uExitReason;
16409	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16410	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16411	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16412	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16413	}
16414
16415	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16416	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16417	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16418	}
16419
16420
16421	/**
16422	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16423	*/
16424	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16425	{
16426	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16427
16428	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16429	{
16430	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16431	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16432	}
16433
16434	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16435	}
16436
16437
16438	/**
16439	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16440	* Conditional VM-exit.
16441	*/
16442	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16443	{
16444	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16445
16446	hmR0VmxReadExitQualVmcs(pVmxTransient);
16447	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16448
16449	VBOXSTRICTRC rcStrict;
16450	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16451	switch (uAccessType)
16452	{
16453	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16454	{
16455	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16456	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16457	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16458	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16459
16460	bool fIntercept;
16461	switch (iCrReg)
16462	{
16463	case 0:
16464	case 4:
16465	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16466	break;
16467
16468	case 3:
16469	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16470	break;
16471
16472	case 8:
16473	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16474	break;
16475
16476	default:
16477	fIntercept = false;
16478	break;
16479	}
16480	if (fIntercept)
16481	{
16482	VMXVEXITINFO ExitInfo;
16483	RT_ZERO(ExitInfo);
16484	ExitInfo.uReason = pVmxTransient->uExitReason;
16485	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16486	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16487	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16488	}
16489	else
16490	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16491	break;
16492	}
16493
16494	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16495	{
16496	/*
16497	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16498	* CR2 reads do not cause a VM-exit.
16499	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16500	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16501	*/
16502	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16503	if ( iCrReg == 3
16504	\|\| iCrReg == 8)
16505	{
16506	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16507	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16508	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16509	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16510	{
16511	VMXVEXITINFO ExitInfo;
16512	RT_ZERO(ExitInfo);
16513	ExitInfo.uReason = pVmxTransient->uExitReason;
16514	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16515	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16516	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16517	}
16518	else
16519	{
16520	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16521	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16522	}
16523	}
16524	else
16525	{
16526	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16527	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16528	}
16529	break;
16530	}
16531
16532	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16533	{
16534	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16535	Assert(pVmcsNstGst);
16536	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16537	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16538	if ( (uGstHostMask & X86_CR0_TS)
16539	&& (uReadShadow & X86_CR0_TS))
16540	{
16541	VMXVEXITINFO ExitInfo;
16542	RT_ZERO(ExitInfo);
16543	ExitInfo.uReason = pVmxTransient->uExitReason;
16544	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16545	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16546	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16547	}
16548	else
16549	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
16550	break;
16551	}
16552
16553	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16554	{
16555	RTGCPTR GCPtrEffDst;
16556	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16557	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16558	if (fMemOperand)
16559	{
16560	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16561	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16562	}
16563	else
16564	GCPtrEffDst = NIL_RTGCPTR;
16565
16566	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16567	{
16568	VMXVEXITINFO ExitInfo;
16569	RT_ZERO(ExitInfo);
16570	ExitInfo.uReason = pVmxTransient->uExitReason;
16571	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16572	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16573	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16574	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16575	}
16576	else
16577	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
16578	break;
16579	}
16580
16581	default:
16582	{
16583	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16584	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16585	}
16586	}
16587
16588	if (rcStrict == VINF_IEM_RAISED_XCPT)
16589	{
16590	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16591	rcStrict = VINF_SUCCESS;
16592	}
16593	return rcStrict;
16594	}
16595
16596
16597	/**
16598	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16599	* Conditional VM-exit.
16600	*/
16601	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16602	{
16603	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16604
16605	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16606	{
16607	hmR0VmxReadExitQualVmcs(pVmxTransient);
16608	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16609
16610	VMXVEXITINFO ExitInfo;
16611	RT_ZERO(ExitInfo);
16612	ExitInfo.uReason = pVmxTransient->uExitReason;
16613	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16614	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16615	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16616	}
16617	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16618	}
16619
16620
16621	/**
16622	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16623	* Conditional VM-exit.
16624	*/
16625	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16626	{
16627	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16628
16629	hmR0VmxReadExitQualVmcs(pVmxTransient);
16630
16631	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16632	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16633	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16634
16635	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16636	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16637	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16638	{
16639	/*
16640	* IN/OUT instruction:
16641	* - Provides VM-exit instruction length.
16642	*
16643	* INS/OUTS instruction:
16644	* - Provides VM-exit instruction length.
16645	* - Provides Guest-linear address.
16646	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16647	*/
16648	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16649	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16650
16651	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16652	pVmxTransient->ExitInstrInfo.u = 0;
16653	pVmxTransient->uGuestLinearAddr = 0;
16654
16655	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16656	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16657	if (fIOString)
16658	{
16659	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16660	if (fVmxInsOutsInfo)
16661	{
16662	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16663	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16664	}
16665	}
16666
16667	VMXVEXITINFO ExitInfo;
16668	RT_ZERO(ExitInfo);
16669	ExitInfo.uReason = pVmxTransient->uExitReason;
16670	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16671	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16672	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16673	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16674	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16675	}
16676	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16677	}
16678
16679
16680	/**
16681	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16682	*/
16683	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16684	{
16685	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16686
16687	uint32_t fMsrpm;
16688	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16689	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16690	else
16691	fMsrpm = VMXMSRPM_EXIT_RD;
16692
16693	if (fMsrpm & VMXMSRPM_EXIT_RD)
16694	{
16695	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16696	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16697	}
16698	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16699	}
16700
16701
16702	/**
16703	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16704	*/
16705	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16706	{
16707	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16708
16709	uint32_t fMsrpm;
16710	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16711	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16712	else
16713	fMsrpm = VMXMSRPM_EXIT_WR;
16714
16715	if (fMsrpm & VMXMSRPM_EXIT_WR)
16716	{
16717	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16718	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16719	}
16720	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16721	}
16722
16723
16724	/**
16725	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16726	*/
16727	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16728	{
16729	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16730
16731	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16732	{
16733	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16734	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16735	}
16736	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16737	}
16738
16739
16740	/**
16741	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16742	* VM-exit.
16743	*/
16744	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16745	{
16746	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16747
16748	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16749	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16750	VMXVEXITINFO ExitInfo;
16751	RT_ZERO(ExitInfo);
16752	ExitInfo.uReason = pVmxTransient->uExitReason;
16753	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16754	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16755	}
16756
16757
16758	/**
16759	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16760	*/
16761	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16762	{
16763	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16764
16765	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16766	{
16767	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16768	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16769	}
16770	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16771	}
16772
16773
16774	/**
16775	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16776	*/
16777	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16778	{
16779	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16780
16781	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16782	* PAUSE when executing a nested-guest? If it does not, we would not need
16783	* to check for the intercepts here. Just call VM-exit... */
16784
16785	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16786	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16787	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16788	{
16789	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16790	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16791	}
16792	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16793	}
16794
16795
16796	/**
16797	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16798	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16799	*/
16800	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16801	{
16802	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16803
16804	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16805	{
16806	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16807	VMXVEXITINFO ExitInfo;
16808	RT_ZERO(ExitInfo);
16809	ExitInfo.uReason = pVmxTransient->uExitReason;
16810	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16811	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16812	}
16813	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16814	}
16815
16816
16817	/**
16818	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16819	* VM-exit.
16820	*/
16821	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16822	{
16823	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16824
16825	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16826	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16827	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16828	hmR0VmxReadExitQualVmcs(pVmxTransient);
16829
16830	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16831
16832	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
16833	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
16834
16835	VMXVEXITINFO ExitInfo;
16836	RT_ZERO(ExitInfo);
16837	ExitInfo.uReason = pVmxTransient->uExitReason;
16838	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16839	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16840
16841	VMXVEXITEVENTINFO ExitEventInfo;
16842	RT_ZERO(ExitEventInfo);
16843	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16844	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16845	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
16846	}
16847
16848
16849	/**
16850	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
16851	* Conditional VM-exit.
16852	*/
16853	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16854	{
16855	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16856
16857	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
16858	hmR0VmxReadExitQualVmcs(pVmxTransient);
16859	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16860	}
16861
16862
16863	/**
16864	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
16865	* Conditional VM-exit.
16866	*/
16867	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16868	{
16869	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16870
16871	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
16872	hmR0VmxReadExitQualVmcs(pVmxTransient);
16873	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16874	}
16875
16876
16877	/**
16878	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16879	*/
16880	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16881	{
16882	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16883
16884	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16885	{
16886	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16887	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16888	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16889	}
16890	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16891	}
16892
16893
16894	/**
16895	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16896	*/
16897	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16898	{
16899	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16900
16901	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16902	{
16903	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16904	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16905	}
16906	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16907	}
16908
16909
16910	/**
16911	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16912	*/
16913	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16914	{
16915	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16916
16917	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16918	{
16919	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
16920	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16921	hmR0VmxReadExitQualVmcs(pVmxTransient);
16922	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16923
16924	VMXVEXITINFO ExitInfo;
16925	RT_ZERO(ExitInfo);
16926	ExitInfo.uReason = pVmxTransient->uExitReason;
16927	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16928	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16929	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16930	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16931	}
16932	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
16933	}
16934
16935
16936	/**
16937	* Nested-guest VM-exit handler for invalid-guest state
16938	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
16939	*/
16940	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16941	{
16942	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16943
16944	/*
16945	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
16946	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
16947	* Handle it like it's in an invalid guest state of the outer guest.
16948	*
16949	* When the fast path is implemented, this should be changed to cause the corresponding
16950	* nested-guest VM-exit.
16951	*/
16952	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
16953	}
16954
16955
16956	/**
16957	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
16958	* and only provide the instruction length.
16959	*
16960	* Unconditional VM-exit.
16961	*/
16962	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16963	{
16964	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16965
16966	#ifdef VBOX_STRICT
16967	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16968	switch (pVmxTransient->uExitReason)
16969	{
16970	case VMX_EXIT_ENCLS:
16971	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
16972	break;
16973
16974	case VMX_EXIT_VMFUNC:
16975	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
16976	break;
16977	}
16978	#endif
16979
16980	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16981	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16982	}
16983
16984
16985	/**
16986	* Nested-guest VM-exit handler for instructions that provide instruction length as
16987	* well as more information.
16988	*
16989	* Unconditional VM-exit.
16990	*/
16991	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16992	{
16993	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16994
16995	#ifdef VBOX_STRICT
16996	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16997	switch (pVmxTransient->uExitReason)
16998	{
16999	case VMX_EXIT_GDTR_IDTR_ACCESS:
17000	case VMX_EXIT_LDTR_TR_ACCESS:
17001	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17002	break;
17003
17004	case VMX_EXIT_RDRAND:
17005	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17006	break;
17007
17008	case VMX_EXIT_RDSEED:
17009	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17010	break;
17011
17012	case VMX_EXIT_XSAVES:
17013	case VMX_EXIT_XRSTORS:
17014	/** @todo NSTVMX: Verify XSS-bitmap. */
17015	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17016	break;
17017
17018	case VMX_EXIT_UMWAIT:
17019	case VMX_EXIT_TPAUSE:
17020	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17021	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17022	break;
17023	}
17024	#endif
17025
17026	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17027	hmR0VmxReadExitQualVmcs(pVmxTransient);
17028	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17029
17030	VMXVEXITINFO ExitInfo;
17031	RT_ZERO(ExitInfo);
17032	ExitInfo.uReason = pVmxTransient->uExitReason;
17033	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17034	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17035	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17036	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17037	}
17038
17039	/** @} */
17040	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17041

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 81239

Download in other formats: