HMVMXR0.cpp@ 81150

Last change on this file since 81150 was 81150, checked in by vboxsync, 5 years ago
VMM,/Makefile.kmk: Kicked out more recompiler related code. bugref:9576
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 710.0 KB

Line
1	/* $Id: HMVMXR0.cpp 81150 2019-10-08 12:53:47Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#include "HMInternal.h"
39	#include <VBox/vmm/vmcc.h>
40	#include <VBox/vmm/hmvmxinline.h>
41	#include "HMVMXR0.h"
42	#include "dtrace/VBoxVMM.h"
43
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
47	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/**
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
83	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
84
85	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
86	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
87	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
88	\| HMVMX_READ_EXIT_INSTR_LEN \
89	\| HMVMX_READ_IDT_VECTORING_INFO \
90	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
91
92	/** Assert that all the given fields have been read from the VMCS. */
93	#ifdef VBOX_STRICT
94	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
95	do { \
96	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
97	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); \
98	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
99	} while (0)
100	#else
101	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
102	#endif
103
104	/**
105	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
106	* guest using hardware-assisted VMX.
107	*
108	* This excludes state like GPRs (other than RSP) which are always are
109	* swapped and restored across the world-switch and also registers like EFER,
110	* MSR which cannot be modified by the guest without causing a VM-exit.
111	*/
112	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
113	\| CPUMCTX_EXTRN_RFLAGS \
114	\| CPUMCTX_EXTRN_RSP \
115	\| CPUMCTX_EXTRN_SREG_MASK \
116	\| CPUMCTX_EXTRN_TABLE_MASK \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_TSC_AUX \
121	\| CPUMCTX_EXTRN_OTHER_MSRS \
122	\| CPUMCTX_EXTRN_CR0 \
123	\| CPUMCTX_EXTRN_CR3 \
124	\| CPUMCTX_EXTRN_CR4 \
125	\| CPUMCTX_EXTRN_DR7 \
126	\| CPUMCTX_EXTRN_HWVIRT \
127	\| CPUMCTX_EXTRN_HM_VMX_MASK)
128
129	/**
130	* Exception bitmap mask for real-mode guests (real-on-v86).
131	*
132	* We need to intercept all exceptions manually except:
133	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
134	* due to bugs in Intel CPUs.
135	* - \#PF need not be intercepted even in real-mode if we have nested paging
136	* support.
137	*/
138	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
139	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
140	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
141	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
142	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
143	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
144	\| RT_BIT(X86_XCPT_XF))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158	/** Assert that preemption is disabled or covered by thread-context hooks. */
159	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
160	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
161
162	/** Assert that we haven't migrated CPUs when thread-context hooks are not
163	* used. */
164	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
165	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
166	("Illegal migration! Entered on CPU %u Current %u\n", \
167	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
168
169	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
170	* context. */
171	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
172	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
173	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
174
175	/** Log the VM-exit reason with an easily visible marker to identify it in a
176	* potential sea of logging data. */
177	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
178	do { \
179	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
180	HMGetVmxExitName(a_uExitReason))); \
181	} while (0) \
182
183
184	/*********************************************************************************************************************************
185	* Structures and Typedefs *
186	*********************************************************************************************************************************/
187	/**
188	* VMX per-VCPU transient state.
189	*
190	* A state structure for holding miscellaneous information across
191	* VMX non-root operation and restored after the transition.
192	*/
193	typedef struct VMXTRANSIENT
194	{
195	/** The host's rflags/eflags. */
196	RTCCUINTREG fEFlags;
197
198	/** The guest's TPR value used for TPR shadowing. */
199	uint8_t u8GuestTpr;
200	/** Alignment. */
201	uint8_t abAlignment0[7];
202
203	/** The basic VM-exit reason. */
204	uint16_t uExitReason;
205	/** Alignment. */
206	uint16_t u16Alignment0;
207	/** The VM-exit interruption error code. */
208	uint32_t uExitIntErrorCode;
209	/** The VM-exit exit code qualification. */
210	uint64_t uExitQual;
211	/** The Guest-linear address. */
212	uint64_t uGuestLinearAddr;
213	/** The Guest-physical address. */
214	uint64_t uGuestPhysicalAddr;
215	/** The Guest pending-debug exceptions. */
216	uint64_t uGuestPendingDbgXcpts;
217
218	/** The VM-exit interruption-information field. */
219	uint32_t uExitIntInfo;
220	/** The VM-exit instruction-length field. */
221	uint32_t cbExitInstr;
222	/** The VM-exit instruction-information field. */
223	VMXEXITINSTRINFO ExitInstrInfo;
224	/** Whether the VM-entry failed or not. */
225	bool fVMEntryFailed;
226	/** Whether we are currently executing a nested-guest. */
227	bool fIsNestedGuest;
228	/** Alignment. */
229	uint8_t abAlignment1[2];
230
231	/** The VM-entry interruption-information field. */
232	uint32_t uEntryIntInfo;
233	/** The VM-entry exception error code field. */
234	uint32_t uEntryXcptErrorCode;
235	/** The VM-entry instruction length field. */
236	uint32_t cbEntryInstr;
237
238	/** IDT-vectoring information field. */
239	uint32_t uIdtVectoringInfo;
240	/** IDT-vectoring error code. */
241	uint32_t uIdtVectoringErrorCode;
242
243	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
244	uint32_t fVmcsFieldsRead;
245
246	/** Whether the guest debug state was active at the time of VM-exit. */
247	bool fWasGuestDebugStateActive;
248	/** Whether the hyper debug state was active at the time of VM-exit. */
249	bool fWasHyperDebugStateActive;
250	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
251	bool fUpdatedTscOffsettingAndPreemptTimer;
252	/** Whether the VM-exit was caused by a page-fault during delivery of a
253	* contributory exception or a page-fault. */
254	bool fVectoringDoublePF;
255	/** Whether the VM-exit was caused by a page-fault during delivery of an
256	* external interrupt or NMI. */
257	bool fVectoringPF;
258	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
259	* area after VM-exit. */
260	bool fRemoveTscAuxMsr;
261	bool afAlignment0[2];
262
263	/** The VMCS info. object. */
264	PVMXVMCSINFO pVmcsInfo;
265	} VMXTRANSIENT;
266	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
267	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
268	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
269	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
271	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
272	/** Pointer to VMX transient state. */
273	typedef VMXTRANSIENT *PVMXTRANSIENT;
274	/** Pointer to a const VMX transient state. */
275	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
276
277	/**
278	* Memory operand read or write access.
279	*/
280	typedef enum VMXMEMACCESS
281	{
282	VMXMEMACCESS_READ = 0,
283	VMXMEMACCESS_WRITE = 1
284	} VMXMEMACCESS;
285
286	/**
287	* VMX VM-exit handler.
288	*
289	* @returns Strict VBox status code (i.e. informational status codes too).
290	* @param pVCpu The cross context virtual CPU structure.
291	* @param pVmxTransient The VMX-transient structure.
292	*/
293	#ifndef HMVMX_USE_FUNCTION_TABLE
294	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
295	#else
296	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
297	/** Pointer to VM-exit handler. */
298	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
299	#endif
300
301	/**
302	* VMX VM-exit handler, non-strict status code.
303	*
304	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
305	*
306	* @returns VBox status code, no informational status code returned.
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*
310	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
311	* use of that status code will be replaced with VINF_EM_SOMETHING
312	* later when switching over to IEM.
313	*/
314	#ifndef HMVMX_USE_FUNCTION_TABLE
315	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
316	#else
317	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
318	#endif
319
320
321	/*********************************************************************************************************************************
322	* Internal Functions *
323	*********************************************************************************************************************************/
324	#ifndef HMVMX_USE_FUNCTION_TABLE
325	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
326	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
327	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
328	#else
329	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
330	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
331	#endif
332	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
333	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
334	#endif
335
336	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
337
338	/** @name VM-exit handler prototypes.
339	* @{
340	*/
341	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
342	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
343	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
344	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
345	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
346	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
347	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
348	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
349	static FNVMXEXITHANDLER hmR0VmxExitHlt;
350	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
351	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
352	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
353	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
354	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
355	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
356	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
357	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
358	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
359	static FNVMXEXITHANDLER hmR0VmxExitVmread;
360	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
361	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
362	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
363	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
364	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
365	#endif
366	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
367	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
368	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
369	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
370	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
371	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
372	static FNVMXEXITHANDLER hmR0VmxExitMwait;
373	static FNVMXEXITHANDLER hmR0VmxExitMtf;
374	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
375	static FNVMXEXITHANDLER hmR0VmxExitPause;
376	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
377	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
378	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
379	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
380	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
381	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
382	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
383	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
384	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
385	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
386	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
388	/** @} */
389
390	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
391	/** @name Nested-guest VM-exit handler prototypes.
392	* @{
393	*/
394	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
395	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
398	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
399	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
400	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
401	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
402	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
403	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
404	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
405	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
406	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
407	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
408	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
409	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
410	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
411	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
412	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
414	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
415	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
416	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
417	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
418	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
419	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
421	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
422	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
423	/** @} */
424	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
425
426
427	/*********************************************************************************************************************************
428	* Global Variables *
429	*********************************************************************************************************************************/
430	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
431	/**
432	* Array of all VMCS fields.
433	* Any fields added to the VT-x spec. should be added here.
434	*
435	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
436	* of nested-guests.
437	*/
438	static const uint32_t g_aVmcsFields[] =
439	{
440	/* 16-bit control fields. */
441	VMX_VMCS16_VPID,
442	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
443	VMX_VMCS16_EPTP_INDEX,
444
445	/* 16-bit guest-state fields. */
446	VMX_VMCS16_GUEST_ES_SEL,
447	VMX_VMCS16_GUEST_CS_SEL,
448	VMX_VMCS16_GUEST_SS_SEL,
449	VMX_VMCS16_GUEST_DS_SEL,
450	VMX_VMCS16_GUEST_FS_SEL,
451	VMX_VMCS16_GUEST_GS_SEL,
452	VMX_VMCS16_GUEST_LDTR_SEL,
453	VMX_VMCS16_GUEST_TR_SEL,
454	VMX_VMCS16_GUEST_INTR_STATUS,
455	VMX_VMCS16_GUEST_PML_INDEX,
456
457	/* 16-bits host-state fields. */
458	VMX_VMCS16_HOST_ES_SEL,
459	VMX_VMCS16_HOST_CS_SEL,
460	VMX_VMCS16_HOST_SS_SEL,
461	VMX_VMCS16_HOST_DS_SEL,
462	VMX_VMCS16_HOST_FS_SEL,
463	VMX_VMCS16_HOST_GS_SEL,
464	VMX_VMCS16_HOST_TR_SEL,
465
466	/* 64-bit control fields. */
467	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
468	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
469	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
470	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
471	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
472	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
473	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
474	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
475	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
476	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
477	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
478	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
479	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
480	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
481	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
482	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
483	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
484	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
485	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
486	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
487	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
488	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
489	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
490	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
491	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
492	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
493	VMX_VMCS64_CTRL_EPTP_FULL,
494	VMX_VMCS64_CTRL_EPTP_HIGH,
495	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
496	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
497	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
498	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
499	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
500	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
501	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
502	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
503	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
504	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
505	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
506	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
507	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
508	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
509	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
510	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
511	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
512	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
513	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
514	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
515	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
516	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
517
518	/* 64-bit read-only data fields. */
519	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
520	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
521
522	/* 64-bit guest-state fields. */
523	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
524	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
525	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
526	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
527	VMX_VMCS64_GUEST_PAT_FULL,
528	VMX_VMCS64_GUEST_PAT_HIGH,
529	VMX_VMCS64_GUEST_EFER_FULL,
530	VMX_VMCS64_GUEST_EFER_HIGH,
531	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
532	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
533	VMX_VMCS64_GUEST_PDPTE0_FULL,
534	VMX_VMCS64_GUEST_PDPTE0_HIGH,
535	VMX_VMCS64_GUEST_PDPTE1_FULL,
536	VMX_VMCS64_GUEST_PDPTE1_HIGH,
537	VMX_VMCS64_GUEST_PDPTE2_FULL,
538	VMX_VMCS64_GUEST_PDPTE2_HIGH,
539	VMX_VMCS64_GUEST_PDPTE3_FULL,
540	VMX_VMCS64_GUEST_PDPTE3_HIGH,
541	VMX_VMCS64_GUEST_BNDCFGS_FULL,
542	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
543
544	/* 64-bit host-state fields. */
545	VMX_VMCS64_HOST_PAT_FULL,
546	VMX_VMCS64_HOST_PAT_HIGH,
547	VMX_VMCS64_HOST_EFER_FULL,
548	VMX_VMCS64_HOST_EFER_HIGH,
549	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
550	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
551
552	/* 32-bit control fields. */
553	VMX_VMCS32_CTRL_PIN_EXEC,
554	VMX_VMCS32_CTRL_PROC_EXEC,
555	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
556	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
557	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
558	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
559	VMX_VMCS32_CTRL_EXIT,
560	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
561	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
562	VMX_VMCS32_CTRL_ENTRY,
563	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
564	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
565	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
566	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
567	VMX_VMCS32_CTRL_TPR_THRESHOLD,
568	VMX_VMCS32_CTRL_PROC_EXEC2,
569	VMX_VMCS32_CTRL_PLE_GAP,
570	VMX_VMCS32_CTRL_PLE_WINDOW,
571
572	/* 32-bits read-only fields. */
573	VMX_VMCS32_RO_VM_INSTR_ERROR,
574	VMX_VMCS32_RO_EXIT_REASON,
575	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
576	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
577	VMX_VMCS32_RO_IDT_VECTORING_INFO,
578	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
579	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
580	VMX_VMCS32_RO_EXIT_INSTR_INFO,
581
582	/* 32-bit guest-state fields. */
583	VMX_VMCS32_GUEST_ES_LIMIT,
584	VMX_VMCS32_GUEST_CS_LIMIT,
585	VMX_VMCS32_GUEST_SS_LIMIT,
586	VMX_VMCS32_GUEST_DS_LIMIT,
587	VMX_VMCS32_GUEST_FS_LIMIT,
588	VMX_VMCS32_GUEST_GS_LIMIT,
589	VMX_VMCS32_GUEST_LDTR_LIMIT,
590	VMX_VMCS32_GUEST_TR_LIMIT,
591	VMX_VMCS32_GUEST_GDTR_LIMIT,
592	VMX_VMCS32_GUEST_IDTR_LIMIT,
593	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
594	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
595	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
596	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
597	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
598	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
599	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
600	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_INT_STATE,
602	VMX_VMCS32_GUEST_ACTIVITY_STATE,
603	VMX_VMCS32_GUEST_SMBASE,
604	VMX_VMCS32_GUEST_SYSENTER_CS,
605	VMX_VMCS32_PREEMPT_TIMER_VALUE,
606
607	/* 32-bit host-state fields. */
608	VMX_VMCS32_HOST_SYSENTER_CS,
609
610	/* Natural-width control fields. */
611	VMX_VMCS_CTRL_CR0_MASK,
612	VMX_VMCS_CTRL_CR4_MASK,
613	VMX_VMCS_CTRL_CR0_READ_SHADOW,
614	VMX_VMCS_CTRL_CR4_READ_SHADOW,
615	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
616	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
617	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
618	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
619
620	/* Natural-width read-only data fields. */
621	VMX_VMCS_RO_EXIT_QUALIFICATION,
622	VMX_VMCS_RO_IO_RCX,
623	VMX_VMCS_RO_IO_RSI,
624	VMX_VMCS_RO_IO_RDI,
625	VMX_VMCS_RO_IO_RIP,
626	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
627
628	/* Natural-width guest-state field */
629	VMX_VMCS_GUEST_CR0,
630	VMX_VMCS_GUEST_CR3,
631	VMX_VMCS_GUEST_CR4,
632	VMX_VMCS_GUEST_ES_BASE,
633	VMX_VMCS_GUEST_CS_BASE,
634	VMX_VMCS_GUEST_SS_BASE,
635	VMX_VMCS_GUEST_DS_BASE,
636	VMX_VMCS_GUEST_FS_BASE,
637	VMX_VMCS_GUEST_GS_BASE,
638	VMX_VMCS_GUEST_LDTR_BASE,
639	VMX_VMCS_GUEST_TR_BASE,
640	VMX_VMCS_GUEST_GDTR_BASE,
641	VMX_VMCS_GUEST_IDTR_BASE,
642	VMX_VMCS_GUEST_DR7,
643	VMX_VMCS_GUEST_RSP,
644	VMX_VMCS_GUEST_RIP,
645	VMX_VMCS_GUEST_RFLAGS,
646	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
647	VMX_VMCS_GUEST_SYSENTER_ESP,
648	VMX_VMCS_GUEST_SYSENTER_EIP,
649
650	/* Natural-width host-state fields */
651	VMX_VMCS_HOST_CR0,
652	VMX_VMCS_HOST_CR3,
653	VMX_VMCS_HOST_CR4,
654	VMX_VMCS_HOST_FS_BASE,
655	VMX_VMCS_HOST_GS_BASE,
656	VMX_VMCS_HOST_TR_BASE,
657	VMX_VMCS_HOST_GDTR_BASE,
658	VMX_VMCS_HOST_IDTR_BASE,
659	VMX_VMCS_HOST_SYSENTER_ESP,
660	VMX_VMCS_HOST_SYSENTER_EIP,
661	VMX_VMCS_HOST_RSP,
662	VMX_VMCS_HOST_RIP
663	};
664	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
665
666	static const uint32_t g_aVmcsSegBase[] =
667	{
668	VMX_VMCS_GUEST_ES_BASE,
669	VMX_VMCS_GUEST_CS_BASE,
670	VMX_VMCS_GUEST_SS_BASE,
671	VMX_VMCS_GUEST_DS_BASE,
672	VMX_VMCS_GUEST_FS_BASE,
673	VMX_VMCS_GUEST_GS_BASE
674	};
675	static const uint32_t g_aVmcsSegSel[] =
676	{
677	VMX_VMCS16_GUEST_ES_SEL,
678	VMX_VMCS16_GUEST_CS_SEL,
679	VMX_VMCS16_GUEST_SS_SEL,
680	VMX_VMCS16_GUEST_DS_SEL,
681	VMX_VMCS16_GUEST_FS_SEL,
682	VMX_VMCS16_GUEST_GS_SEL
683	};
684	static const uint32_t g_aVmcsSegLimit[] =
685	{
686	VMX_VMCS32_GUEST_ES_LIMIT,
687	VMX_VMCS32_GUEST_CS_LIMIT,
688	VMX_VMCS32_GUEST_SS_LIMIT,
689	VMX_VMCS32_GUEST_DS_LIMIT,
690	VMX_VMCS32_GUEST_FS_LIMIT,
691	VMX_VMCS32_GUEST_GS_LIMIT
692	};
693	static const uint32_t g_aVmcsSegAttr[] =
694	{
695	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
696	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
697	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
698	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
699	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
700	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
701	};
702	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
703	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
704	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
705	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
706
707	#ifdef HMVMX_USE_FUNCTION_TABLE
708	/**
709	* VMX_EXIT dispatch table.
710	*/
711	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
712	{
713	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
714	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
715	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
716	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
717	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
718	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
719	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
720	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
721	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
722	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
723	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
724	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
725	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
726	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
727	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
728	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
729	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
730	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
731	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
732	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
733	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
734	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
735	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
736	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
737	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
738	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
739	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
740	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
741	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
742	#else
743	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
744	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
745	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
746	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
747	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
748	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
749	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
750	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
751	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
752	#endif
753	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
754	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
755	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
756	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
757	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
758	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
759	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
760	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
761	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
762	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
763	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
764	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
765	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
766	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
767	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
768	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
769	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
770	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
771	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
772	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
773	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
774	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
775	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
776	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
777	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
778	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
779	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
780	#else
781	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
782	#endif
783	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
784	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
785	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
786	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
787	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
788	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
789	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
790	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
791	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
792	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
793	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
794	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
795	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
796	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
797	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
798	};
799	#endif /* HMVMX_USE_FUNCTION_TABLE */
800
801	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
802	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
803	{
804	/* 0 */ "(Not Used)",
805	/* 1 */ "VMCALL executed in VMX root operation.",
806	/* 2 */ "VMCLEAR with invalid physical address.",
807	/* 3 */ "VMCLEAR with VMXON pointer.",
808	/* 4 */ "VMLAUNCH with non-clear VMCS.",
809	/* 5 */ "VMRESUME with non-launched VMCS.",
810	/* 6 */ "VMRESUME after VMXOFF",
811	/* 7 */ "VM-entry with invalid control fields.",
812	/* 8 */ "VM-entry with invalid host state fields.",
813	/* 9 */ "VMPTRLD with invalid physical address.",
814	/* 10 */ "VMPTRLD with VMXON pointer.",
815	/* 11 */ "VMPTRLD with incorrect revision identifier.",
816	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
817	/* 13 */ "VMWRITE to read-only VMCS component.",
818	/* 14 */ "(Not Used)",
819	/* 15 */ "VMXON executed in VMX root operation.",
820	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
821	/* 17 */ "VM-entry with non-launched executing VMCS.",
822	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
823	/* 19 */ "VMCALL with non-clear VMCS.",
824	/* 20 */ "VMCALL with invalid VM-exit control fields.",
825	/* 21 */ "(Not Used)",
826	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
827	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
828	/* 24 */ "VMCALL with invalid SMM-monitor features.",
829	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
830	/* 26 */ "VM-entry with events blocked by MOV SS.",
831	/* 27 */ "(Not Used)",
832	/* 28 */ "Invalid operand to INVEPT/INVVPID."
833	};
834	#endif /* VBOX_STRICT && LOG_ENABLED */
835
836
837	/**
838	* Gets the CR0 guest/host mask.
839	*
840	* These bits typically does not change through the lifetime of a VM. Any bit set in
841	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
842	* by the guest.
843	*
844	* @returns The CR0 guest/host mask.
845	* @param pVCpu The cross context virtual CPU structure.
846	*/
847	static uint64_t hmR0VmxGetFixedCr0Mask(PCVMCPUCC pVCpu)
848	{
849	/*
850	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
851	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
852	*
853	* Furthermore, modifications to any bits that are reserved/unspecified currently
854	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
855	* when future CPUs specify and use currently reserved/unspecified bits.
856	*/
857	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
858	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
859	* and @bugref{6944}. */
860	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
861	return ( X86_CR0_PE
862	\| X86_CR0_NE
863	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
864	\| X86_CR0_PG
865	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
866	}
867
868
869	/**
870	* Gets the CR4 guest/host mask.
871	*
872	* These bits typically does not change through the lifetime of a VM. Any bit set in
873	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
874	* by the guest.
875	*
876	* @returns The CR4 guest/host mask.
877	* @param pVCpu The cross context virtual CPU structure.
878	*/
879	static uint64_t hmR0VmxGetFixedCr4Mask(PCVMCPUCC pVCpu)
880	{
881	/*
882	* We construct a mask of all CR4 bits that the guest can modify without causing
883	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
884	* a VM-exit when the guest attempts to modify them when executing using
885	* hardware-assisted VMX.
886	*
887	* When a feature is not exposed to the guest (and may be present on the host),
888	* we want to intercept guest modifications to the bit so we can emulate proper
889	* behavior (e.g., #GP).
890	*
891	* Furthermore, only modifications to those bits that don't require immediate
892	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
893	* depends on CR3 which might not always be the guest value while executing
894	* using hardware-assisted VMX.
895	*/
896	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
897	bool const fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
898	bool const fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
899	bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
900
901	/*
902	* Paranoia.
903	* Ensure features exposed to the guest are present on the host.
904	*/
905	Assert(!fFsGsBase \|\| pVM->cpum.ro.HostFeatures.fFsGsBase);
906	Assert(!fXSaveRstor \|\| pVM->cpum.ro.HostFeatures.fXSaveRstor);
907	Assert(!fFxSaveRstor \|\| pVM->cpum.ro.HostFeatures.fFxSaveRstor);
908
909	uint64_t const fGstMask = ( X86_CR4_PVI
910	\| X86_CR4_TSD
911	\| X86_CR4_DE
912	\| X86_CR4_MCE
913	\| X86_CR4_PCE
914	\| X86_CR4_OSXMMEEXCPT
915	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
916	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
917	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0));
918	return ~fGstMask;
919	}
920
921
922	/**
923	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
924	* area.
925	*
926	* @returns @c true if it's different, @c false otherwise.
927	* @param pVmcsInfo The VMCS info. object.
928	*/
929	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
930	{
931	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
932	&& pVmcsInfo->pvGuestMsrStore);
933	}
934
935
936	/**
937	* Sets the given Processor-based VM-execution controls.
938	*
939	* @param pVmxTransient The VMX-transient structure.
940	* @param uProcCtls The Processor-based VM-execution controls to set.
941	*/
942	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
943	{
944	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
945	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
946	{
947	pVmcsInfo->u32ProcCtls \|= uProcCtls;
948	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
949	AssertRC(rc);
950	}
951	}
952
953
954	/**
955	* Removes the given Processor-based VM-execution controls.
956	*
957	* @param pVCpu The cross context virtual CPU structure.
958	* @param pVmxTransient The VMX-transient structure.
959	* @param uProcCtls The Processor-based VM-execution controls to remove.
960	*
961	* @remarks When executing a nested-guest, this will not remove any of the specified
962	* controls if the nested hypervisor has set any one of them.
963	*/
964	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
965	{
966	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
967	if (pVmcsInfo->u32ProcCtls & uProcCtls)
968	{
969	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
970	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
971	? true
972	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
973	#else
974	NOREF(pVCpu);
975	bool const fRemoveCtls = true;
976	#endif
977	if (fRemoveCtls)
978	{
979	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
980	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
981	AssertRC(rc);
982	}
983	}
984	}
985
986
987	/**
988	* Sets the TSC offset for the current VMCS.
989	*
990	* @param uTscOffset The TSC offset to set.
991	* @param pVmcsInfo The VMCS info. object.
992	*/
993	static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
994	{
995	if (pVmcsInfo->u64TscOffset != uTscOffset)
996	{
997	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
998	AssertRC(rc);
999	pVmcsInfo->u64TscOffset = uTscOffset;
1000	}
1001	}
1002
1003
1004	/**
1005	* Adds one or more exceptions to the exception bitmap and commits it to the current
1006	* VMCS.
1007	*
1008	* @param pVmxTransient The VMX-transient structure.
1009	* @param uXcptMask The exception(s) to add.
1010	*/
1011	static void hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1012	{
1013	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1014	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1015	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1016	{
1017	uXcptBitmap \|= uXcptMask;
1018	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1019	AssertRC(rc);
1020	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1021	}
1022	}
1023
1024
1025	/**
1026	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1027	*
1028	* @param pVmxTransient The VMX-transient structure.
1029	* @param uXcpt The exception to add.
1030	*/
1031	static void hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1032	{
1033	Assert(uXcpt <= X86_XCPT_LAST);
1034	hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1035	}
1036
1037
1038	/**
1039	* Remove one or more exceptions from the exception bitmap and commits it to the
1040	* current VMCS.
1041	*
1042	* This takes care of not removing the exception intercept if a nested-guest
1043	* requires the exception to be intercepted.
1044	*
1045	* @returns VBox status code.
1046	* @param pVCpu The cross context virtual CPU structure.
1047	* @param pVmxTransient The VMX-transient structure.
1048	* @param uXcptMask The exception(s) to remove.
1049	*/
1050	static int hmR0VmxRemoveXcptInterceptMask(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1051	{
1052	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1053	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1054	if (u32XcptBitmap & uXcptMask)
1055	{
1056	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1057	if (!pVmxTransient->fIsNestedGuest)
1058	{ /* likely */ }
1059	else
1060	{
1061	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1062	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1063	}
1064	#endif
1065	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1066	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1067	\| RT_BIT(X86_XCPT_DE)
1068	\| RT_BIT(X86_XCPT_NM)
1069	\| RT_BIT(X86_XCPT_TS)
1070	\| RT_BIT(X86_XCPT_UD)
1071	\| RT_BIT(X86_XCPT_NP)
1072	\| RT_BIT(X86_XCPT_SS)
1073	\| RT_BIT(X86_XCPT_GP)
1074	\| RT_BIT(X86_XCPT_PF)
1075	\| RT_BIT(X86_XCPT_MF));
1076	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1077	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1078	#endif
1079	if (uXcptMask)
1080	{
1081	/* Validate we are not removing any essential exception intercepts. */
1082	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1083	NOREF(pVCpu);
1084	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1085	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1086
1087	/* Remove it from the exception bitmap. */
1088	u32XcptBitmap &= ~uXcptMask;
1089
1090	/* Commit and update the cache if necessary. */
1091	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1092	{
1093	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1094	AssertRC(rc);
1095	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1096	}
1097	}
1098	}
1099	return VINF_SUCCESS;
1100	}
1101
1102
1103	/**
1104	* Remove an exceptions from the exception bitmap and commits it to the current
1105	* VMCS.
1106	*
1107	* @returns VBox status code.
1108	* @param pVCpu The cross context virtual CPU structure.
1109	* @param pVmxTransient The VMX-transient structure.
1110	* @param uXcpt The exception to remove.
1111	*/
1112	static int hmR0VmxRemoveXcptIntercept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1113	{
1114	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1115	}
1116
1117
1118	/**
1119	* Loads the VMCS specified by the VMCS info. object.
1120	*
1121	* @returns VBox status code.
1122	* @param pVmcsInfo The VMCS info. object.
1123	*
1124	* @remarks Can be called with interrupts disabled.
1125	*/
1126	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1127	{
1128	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1129	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1130
1131	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1132	if (RT_SUCCESS(rc))
1133	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1134	return rc;
1135	}
1136
1137
1138	/**
1139	* Clears the VMCS specified by the VMCS info. object.
1140	*
1141	* @returns VBox status code.
1142	* @param pVmcsInfo The VMCS info. object.
1143	*
1144	* @remarks Can be called with interrupts disabled.
1145	*/
1146	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1147	{
1148	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1150
1151	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1152	if (RT_SUCCESS(rc))
1153	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1154	return rc;
1155	}
1156
1157
1158	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1159	/**
1160	* Loads the shadow VMCS specified by the VMCS info. object.
1161	*
1162	* @returns VBox status code.
1163	* @param pVmcsInfo The VMCS info. object.
1164	*
1165	* @remarks Can be called with interrupts disabled.
1166	*/
1167	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1168	{
1169	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1170	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1171
1172	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1173	if (RT_SUCCESS(rc))
1174	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1175	return rc;
1176	}
1177
1178
1179	/**
1180	* Clears the shadow VMCS specified by the VMCS info. object.
1181	*
1182	* @returns VBox status code.
1183	* @param pVmcsInfo The VMCS info. object.
1184	*
1185	* @remarks Can be called with interrupts disabled.
1186	*/
1187	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1188	{
1189	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1190	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1191
1192	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1193	if (RT_SUCCESS(rc))
1194	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1195	return rc;
1196	}
1197
1198
1199	/**
1200	* Switches from and to the specified VMCSes.
1201	*
1202	* @returns VBox status code.
1203	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1204	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1205	*
1206	* @remarks Called with interrupts disabled.
1207	*/
1208	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1209	{
1210	/*
1211	* Clear the VMCS we are switching out if it has not already been cleared.
1212	* This will sync any CPU internal data back to the VMCS.
1213	*/
1214	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1215	{
1216	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1217	if (RT_SUCCESS(rc))
1218	{
1219	/*
1220	* The shadow VMCS, if any, would not be active at this point since we
1221	* would have cleared it while importing the virtual hardware-virtualization
1222	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1223	* clear the shadow VMCS here, just assert for safety.
1224	*/
1225	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1226	}
1227	else
1228	return rc;
1229	}
1230
1231	/*
1232	* Clear the VMCS we are switching to if it has not already been cleared.
1233	* This will initialize the VMCS launch state to "clear" required for loading it.
1234	*
1235	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1236	*/
1237	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1238	{
1239	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1240	if (RT_SUCCESS(rc))
1241	{ /* likely */ }
1242	else
1243	return rc;
1244	}
1245
1246	/*
1247	* Finally, load the VMCS we are switching to.
1248	*/
1249	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1250	}
1251
1252
1253	/**
1254	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1255	* caller.
1256	*
1257	* @returns VBox status code.
1258	* @param pVCpu The cross context virtual CPU structure.
1259	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1260	* true) or guest VMCS (pass false).
1261	*/
1262	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1263	{
1264	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1265	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1266
1267	PVMXVMCSINFO pVmcsInfoFrom;
1268	PVMXVMCSINFO pVmcsInfoTo;
1269	if (fSwitchToNstGstVmcs)
1270	{
1271	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1272	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1273	}
1274	else
1275	{
1276	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1277	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1278	}
1279
1280	/*
1281	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1282	* preemption hook code path acquires the current VMCS.
1283	*/
1284	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1285
1286	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1287	if (RT_SUCCESS(rc))
1288	{
1289	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1290
1291	/*
1292	* If we are switching to a VMCS that was executed on a different host CPU or was
1293	* never executed before, flag that we need to export the host state before executing
1294	* guest/nested-guest code using hardware-assisted VMX.
1295	*
1296	* This could probably be done in a preemptible context since the preemption hook
1297	* will flag the necessary change in host context. However, since preemption is
1298	* already disabled and to avoid making assumptions about host specific code in
1299	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1300	* disabled.
1301	*/
1302	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1303	{ /* likely */ }
1304	else
1305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1306
1307	ASMSetFlags(fEFlags);
1308
1309	/*
1310	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1311	* flag that we need to update the host MSR values there. Even if we decide in the
1312	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1313	* if its content differs, we would have to update the host MSRs anyway.
1314	*/
1315	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1316	}
1317	else
1318	ASMSetFlags(fEFlags);
1319	return rc;
1320	}
1321	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1322
1323
1324	/**
1325	* Updates the VM's last error record.
1326	*
1327	* If there was a VMX instruction error, reads the error data from the VMCS and
1328	* updates VCPU's last error record as well.
1329	*
1330	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1331	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1332	* VERR_VMX_INVALID_VMCS_FIELD.
1333	* @param rc The error code.
1334	*/
1335	static void hmR0VmxUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
1336	{
1337	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1338	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1339	{
1340	AssertPtrReturnVoid(pVCpu);
1341	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1342	}
1343	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1344	}
1345
1346
1347	#ifdef VBOX_STRICT
1348	/**
1349	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1350	* transient structure.
1351	*
1352	* @param pVmxTransient The VMX-transient structure.
1353	*/
1354	DECLINLINE(void) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1355	{
1356	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1357	AssertRC(rc);
1358	}
1359
1360
1361	/**
1362	* Reads the VM-entry exception error code field from the VMCS into
1363	* the VMX transient structure.
1364	*
1365	* @param pVmxTransient The VMX-transient structure.
1366	*/
1367	DECLINLINE(void) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1368	{
1369	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1370	AssertRC(rc);
1371	}
1372
1373
1374	/**
1375	* Reads the VM-entry exception error code field from the VMCS into
1376	* the VMX transient structure.
1377	*
1378	* @param pVmxTransient The VMX-transient structure.
1379	*/
1380	DECLINLINE(void) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1381	{
1382	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1383	AssertRC(rc);
1384	}
1385	#endif /* VBOX_STRICT */
1386
1387
1388	/**
1389	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1390	* transient structure.
1391	*
1392	* @param pVmxTransient The VMX-transient structure.
1393	*/
1394	DECLINLINE(void) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1395	{
1396	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1397	{
1398	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1399	AssertRC(rc);
1400	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1401	}
1402	}
1403
1404
1405	/**
1406	* Reads the VM-exit interruption error code from the VMCS into the VMX
1407	* transient structure.
1408	*
1409	* @param pVmxTransient The VMX-transient structure.
1410	*/
1411	DECLINLINE(void) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1412	{
1413	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1414	{
1415	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1416	AssertRC(rc);
1417	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1418	}
1419	}
1420
1421
1422	/**
1423	* Reads the VM-exit instruction length field from the VMCS into the VMX
1424	* transient structure.
1425	*
1426	* @param pVmxTransient The VMX-transient structure.
1427	*/
1428	DECLINLINE(void) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1429	{
1430	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1431	{
1432	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1433	AssertRC(rc);
1434	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1435	}
1436	}
1437
1438
1439	/**
1440	* Reads the VM-exit instruction-information field from the VMCS into
1441	* the VMX transient structure.
1442	*
1443	* @param pVmxTransient The VMX-transient structure.
1444	*/
1445	DECLINLINE(void) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1446	{
1447	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1448	{
1449	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1450	AssertRC(rc);
1451	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1452	}
1453	}
1454
1455
1456	/**
1457	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1458	*
1459	* @param pVmxTransient The VMX-transient structure.
1460	*/
1461	DECLINLINE(void) hmR0VmxReadExitQualVmcs(PVMXTRANSIENT pVmxTransient)
1462	{
1463	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1464	{
1465	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1466	AssertRC(rc);
1467	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1468	}
1469	}
1470
1471
1472	/**
1473	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1474	*
1475	* @param pVmxTransient The VMX-transient structure.
1476	*/
1477	DECLINLINE(void) hmR0VmxReadGuestLinearAddrVmcs(PVMXTRANSIENT pVmxTransient)
1478	{
1479	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1480	{
1481	int rc = VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1482	AssertRC(rc);
1483	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1484	}
1485	}
1486
1487
1488	/**
1489	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1490	*
1491	* @param pVmxTransient The VMX-transient structure.
1492	*/
1493	DECLINLINE(void) hmR0VmxReadGuestPhysicalAddrVmcs(PVMXTRANSIENT pVmxTransient)
1494	{
1495	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1496	{
1497	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1498	AssertRC(rc);
1499	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1500	}
1501	}
1502
1503	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1504	/**
1505	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1506	* structure.
1507	*
1508	* @param pVmxTransient The VMX-transient structure.
1509	*/
1510	DECLINLINE(void) hmR0VmxReadGuestPendingDbgXctps(PVMXTRANSIENT pVmxTransient)
1511	{
1512	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1513	{
1514	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1515	AssertRC(rc);
1516	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1517	}
1518	}
1519	#endif
1520
1521	/**
1522	* Reads the IDT-vectoring information field from the VMCS into the VMX
1523	* transient structure.
1524	*
1525	* @param pVmxTransient The VMX-transient structure.
1526	*
1527	* @remarks No-long-jump zone!!!
1528	*/
1529	DECLINLINE(void) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1530	{
1531	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1532	{
1533	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1534	AssertRC(rc);
1535	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1536	}
1537	}
1538
1539
1540	/**
1541	* Reads the IDT-vectoring error code from the VMCS into the VMX
1542	* transient structure.
1543	*
1544	* @param pVmxTransient The VMX-transient structure.
1545	*/
1546	DECLINLINE(void) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1547	{
1548	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1549	{
1550	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1551	AssertRC(rc);
1552	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1553	}
1554	}
1555
1556	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1557	/**
1558	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1559	*
1560	* @param pVmxTransient The VMX-transient structure.
1561	*/
1562	static void hmR0VmxReadAllRoFieldsVmcs(PVMXTRANSIENT pVmxTransient)
1563	{
1564	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1565	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1566	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1567	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1568	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1569	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1570	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1571	rc \|= VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1572	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1573	AssertRC(rc);
1574	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1575	\| HMVMX_READ_EXIT_INSTR_LEN
1576	\| HMVMX_READ_EXIT_INSTR_INFO
1577	\| HMVMX_READ_IDT_VECTORING_INFO
1578	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1579	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1580	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1581	\| HMVMX_READ_GUEST_LINEAR_ADDR
1582	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1583	}
1584	#endif
1585
1586	/**
1587	* Enters VMX root mode operation on the current CPU.
1588	*
1589	* @returns VBox status code.
1590	* @param pHostCpu The HM physical-CPU structure.
1591	* @param pVM The cross context VM structure. Can be
1592	* NULL, after a resume.
1593	* @param HCPhysCpuPage Physical address of the VMXON region.
1594	* @param pvCpuPage Pointer to the VMXON region.
1595	*/
1596	static int hmR0VmxEnterRootMode(PHMPHYSCPU pHostCpu, PVMCC pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1597	{
1598	Assert(pHostCpu);
1599	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1600	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1601	Assert(pvCpuPage);
1602	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1603
1604	if (pVM)
1605	{
1606	/* Write the VMCS revision identifier to the VMXON region. */
1607	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1608	}
1609
1610	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1611	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1612
1613	/* Enable the VMX bit in CR4 if necessary. */
1614	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1615
1616	/* Record whether VMXE was already prior to us enabling it above. */
1617	pHostCpu->fVmxeAlreadyEnabled = RT_BOOL(uOldCr4 & X86_CR4_VMXE);
1618
1619	/* Enter VMX root mode. */
1620	int rc = VMXEnable(HCPhysCpuPage);
1621	if (RT_FAILURE(rc))
1622	{
1623	/* Restore CR4.VMXE if it was not set prior to our attempt to set it above. */
1624	if (!pHostCpu->fVmxeAlreadyEnabled)
1625	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1626
1627	if (pVM)
1628	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1629	}
1630
1631	/* Restore interrupts. */
1632	ASMSetFlags(fEFlags);
1633	return rc;
1634	}
1635
1636
1637	/**
1638	* Exits VMX root mode operation on the current CPU.
1639	*
1640	* @returns VBox status code.
1641	* @param pHostCpu The HM physical-CPU structure.
1642	*/
1643	static int hmR0VmxLeaveRootMode(PHMPHYSCPU pHostCpu)
1644	{
1645	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1646
1647	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1648	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1649
1650	/* If we're for some reason not in VMX root mode, then don't leave it. */
1651	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1652
1653	int rc;
1654	if (uHostCr4 & X86_CR4_VMXE)
1655	{
1656	/* Exit VMX root mode and clear the VMX bit in CR4. */
1657	VMXDisable();
1658
1659	/* Clear CR4.VMXE only if it was clear prior to use setting it. */
1660	if (!pHostCpu->fVmxeAlreadyEnabled)
1661	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1662
1663	rc = VINF_SUCCESS;
1664	}
1665	else
1666	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1667
1668	/* Restore interrupts. */
1669	ASMSetFlags(fEFlags);
1670	return rc;
1671	}
1672
1673
1674	/**
1675	* Allocates and maps a physically contiguous page. The allocated page is
1676	* zero'd out (used by various VT-x structures).
1677	*
1678	* @returns IPRT status code.
1679	* @param pMemObj Pointer to the ring-0 memory object.
1680	* @param ppVirt Where to store the virtual address of the allocation.
1681	* @param pHCPhys Where to store the physical address of the allocation.
1682	*/
1683	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1684	{
1685	AssertPtr(pMemObj);
1686	AssertPtr(ppVirt);
1687	AssertPtr(pHCPhys);
1688	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1689	if (RT_FAILURE(rc))
1690	return rc;
1691	ppVirt = RTR0MemObjAddress(pMemObj);
1692	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1693	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1694	return VINF_SUCCESS;
1695	}
1696
1697
1698	/**
1699	* Frees and unmaps an allocated, physical page.
1700	*
1701	* @param pMemObj Pointer to the ring-0 memory object.
1702	* @param ppVirt Where to re-initialize the virtual address of allocation as
1703	* 0.
1704	* @param pHCPhys Where to re-initialize the physical address of the
1705	* allocation as 0.
1706	*/
1707	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1708	{
1709	AssertPtr(pMemObj);
1710	AssertPtr(ppVirt);
1711	AssertPtr(pHCPhys);
1712	/* NULL is valid, accepted and ignored by the free function below. */
1713	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1714	*pMemObj = NIL_RTR0MEMOBJ;
1715	*ppVirt = NULL;
1716	*pHCPhys = NIL_RTHCPHYS;
1717	}
1718
1719
1720	/**
1721	* Initializes a VMCS info. object.
1722	*
1723	* @param pVmcsInfo The VMCS info. object.
1724	*/
1725	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1726	{
1727	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1728
1729	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1730	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1731	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1732	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1733	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1734	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1735	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1736	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1737	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1738	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1739	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1740	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1741	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1742	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1743	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1744	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
1745	pVmcsInfo->idHostCpuExec = NIL_RTCPUID;
1746	}
1747
1748
1749	/**
1750	* Frees the VT-x structures for a VMCS info. object.
1751	*
1752	* @param pVM The cross context VM structure.
1753	* @param pVmcsInfo The VMCS info. object.
1754	*/
1755	static void hmR0VmxFreeVmcsInfo(PVMCC pVM, PVMXVMCSINFO pVmcsInfo)
1756	{
1757	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1758
1759	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1760	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1761	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1762	#endif
1763
1764	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1765	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1766
1767	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1768	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1769	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1770
1771	hmR0VmxInitVmcsInfo(pVmcsInfo);
1772	}
1773
1774
1775	/**
1776	* Allocates the VT-x structures for a VMCS info. object.
1777	*
1778	* @returns VBox status code.
1779	* @param pVCpu The cross context virtual CPU structure.
1780	* @param pVmcsInfo The VMCS info. object.
1781	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1782	*/
1783	static int hmR0VmxAllocVmcsInfo(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1784	{
1785	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1786
1787	/* Allocate the guest VM control structure (VMCS). */
1788	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1789	if (RT_SUCCESS(rc))
1790	{
1791	if (!fIsNstGstVmcs)
1792	{
1793	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1794	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1795	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1796	#endif
1797	if (RT_SUCCESS(rc))
1798	{
1799	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1800	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1801	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1802	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1803	}
1804	}
1805	else
1806	{
1807	/* We don't yet support exposing VMCS shadowing to the guest. */
1808	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1809	Assert(!pVmcsInfo->pvShadowVmcs);
1810
1811	/* Get the allocated virtual-APIC page from CPUM. */
1812	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1813	{
1814	/** @todo NSTVMX: Get rid of this. There is no need to allocate a separate HC
1815	* page for this. Use the one provided by the nested-guest directly. */
1816	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1817	&pVmcsInfo->HCPhysVirtApic);
1818	Assert(pVmcsInfo->pbVirtApic);
1819	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1820	}
1821	}
1822
1823	if (RT_SUCCESS(rc))
1824	{
1825	/*
1826	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1827	* transparent accesses of specific MSRs.
1828	*
1829	* If the condition for enabling MSR bitmaps changes here, don't forget to
1830	* update HMIsMsrBitmapActive().
1831	*
1832	* We don't share MSR bitmaps between the guest and nested-guest as we then
1833	* don't need to care about carefully restoring the guest MSR bitmap.
1834	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1835	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1836	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1837	* we do that later while merging VMCS.
1838	*/
1839	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1840	{
1841	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1842	if ( RT_SUCCESS(rc)
1843	&& !fIsNstGstVmcs)
1844	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1845	}
1846
1847	if (RT_SUCCESS(rc))
1848	{
1849	/*
1850	* Allocate the VM-entry MSR-load area for the guest MSRs.
1851	*
1852	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1853	* the guest and nested-guest.
1854	*/
1855	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1856	&pVmcsInfo->HCPhysGuestMsrLoad);
1857	if (RT_SUCCESS(rc))
1858	{
1859	/*
1860	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1861	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1862	*/
1863	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1864	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1865	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1866
1867	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1868	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1869	&pVmcsInfo->HCPhysHostMsrLoad);
1870	}
1871	}
1872	}
1873	}
1874
1875	return rc;
1876	}
1877
1878
1879	/**
1880	* Free all VT-x structures for the VM.
1881	*
1882	* @returns IPRT status code.
1883	* @param pVM The cross context VM structure.
1884	*/
1885	static void hmR0VmxStructsFree(PVMCC pVM)
1886	{
1887	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1888	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1889	#endif
1890	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1891
1892	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1893	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1894	{
1895	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1896	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1897	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1898	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1899	}
1900	#endif
1901
1902	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1903	{
1904	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1905	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1906	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1907	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1908	if (pVM->cpum.ro.GuestFeatures.fVmx)
1909	{
1910	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1911	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1912	}
1913	#endif
1914	}
1915	}
1916
1917
1918	/**
1919	* Allocate all VT-x structures for the VM.
1920	*
1921	* @returns IPRT status code.
1922	* @param pVM The cross context VM structure.
1923	*/
1924	static int hmR0VmxStructsAlloc(PVMCC pVM)
1925	{
1926	/*
1927	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1928	* The VMCS size cannot be more than 4096 bytes.
1929	*
1930	* See Intel spec. Appendix A.1 "Basic VMX Information".
1931	*/
1932	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1933	if (cbVmcs <= X86_PAGE_4K_SIZE)
1934	{ /* likely */ }
1935	else
1936	{
1937	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1938	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1939	}
1940
1941	/*
1942	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1943	*/
1944	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1945	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1946	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1947	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1948	#endif
1949
1950	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1951	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1952	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1953
1954	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1955	{
1956	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1957	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1958	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1959	}
1960
1961	/*
1962	* Allocate per-VM VT-x structures.
1963	*/
1964	int rc = VINF_SUCCESS;
1965	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1966	/* Allocate crash-dump magic scratch page. */
1967	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1968	if (RT_FAILURE(rc))
1969	{
1970	hmR0VmxStructsFree(pVM);
1971	return rc;
1972	}
1973	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1974	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1975	#endif
1976
1977	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1978	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1979	{
1980	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1981	&pVM->hm.s.vmx.HCPhysApicAccess);
1982	if (RT_FAILURE(rc))
1983	{
1984	hmR0VmxStructsFree(pVM);
1985	return rc;
1986	}
1987	}
1988
1989	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1990	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1991	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1992	{
1993	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1994	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1995	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1996	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1997	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1998	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1999	{
2000	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
2001	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
2002	if (RT_SUCCESS(rc))
2003	{
2004	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
2005	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
2006	}
2007	}
2008	else
2009	rc = VERR_NO_MEMORY;
2010
2011	if (RT_FAILURE(rc))
2012	{
2013	hmR0VmxStructsFree(pVM);
2014	return rc;
2015	}
2016	}
2017	#endif
2018
2019	/*
2020	* Initialize per-VCPU VT-x structures.
2021	*/
2022	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2023	{
2024	/* Allocate the guest VMCS structures. */
2025	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
2026	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2027	if (RT_SUCCESS(rc))
2028	{
2029	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2030	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2031	if (pVM->cpum.ro.GuestFeatures.fVmx)
2032	{
2033	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2034	if (RT_SUCCESS(rc))
2035	{ /* likely */ }
2036	else
2037	break;
2038	}
2039	#endif
2040	}
2041	else
2042	break;
2043	}
2044
2045	if (RT_FAILURE(rc))
2046	{
2047	hmR0VmxStructsFree(pVM);
2048	return rc;
2049	}
2050
2051	return VINF_SUCCESS;
2052	}
2053
2054	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2055	/**
2056	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2057	*
2058	* @returns @c true if the MSR is intercepted, @c false otherwise.
2059	* @param pvMsrBitmap The MSR bitmap.
2060	* @param offMsr The MSR byte offset.
2061	* @param iBit The bit offset from the byte offset.
2062	*/
2063	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2064	{
2065	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2066	Assert(pbMsrBitmap);
2067	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2068	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2069	}
2070	#endif
2071
2072	/**
2073	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2074	*
2075	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2076	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2077	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2078	* the read/write access of this MSR.
2079	*
2080	* @param pVCpu The cross context virtual CPU structure.
2081	* @param pVmcsInfo The VMCS info. object.
2082	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2083	* @param idMsr The MSR value.
2084	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2085	* include both a read -and- a write permission!
2086	*
2087	* @sa CPUMGetVmxMsrPermission.
2088	* @remarks Can be called with interrupts disabled.
2089	*/
2090	static void hmR0VmxSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2091	{
2092	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2093	Assert(pbMsrBitmap);
2094	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2095
2096	/*
2097	* MSR-bitmap Layout:
2098	* Byte index MSR range Interpreted as
2099	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2100	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2101	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2102	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2103	*
2104	* A bit corresponding to an MSR within the above range causes a VM-exit
2105	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2106	* the MSR range, it always cause a VM-exit.
2107	*
2108	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2109	*/
2110	uint16_t const offBitmapRead = 0;
2111	uint16_t const offBitmapWrite = 0x800;
2112	uint16_t offMsr;
2113	int32_t iBit;
2114	if (idMsr <= UINT32_C(0x00001fff))
2115	{
2116	offMsr = 0;
2117	iBit = idMsr;
2118	}
2119	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2120	{
2121	offMsr = 0x400;
2122	iBit = idMsr - UINT32_C(0xc0000000);
2123	}
2124	else
2125	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2126
2127	/*
2128	* Set the MSR read permission.
2129	*/
2130	uint16_t const offMsrRead = offBitmapRead + offMsr;
2131	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2132	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2133	{
2134	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2135	bool const fClear = !fIsNstGstVmcs ? true
2136	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2137	#else
2138	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2139	bool const fClear = true;
2140	#endif
2141	if (fClear)
2142	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2143	}
2144	else
2145	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2146
2147	/*
2148	* Set the MSR write permission.
2149	*/
2150	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2151	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2152	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2153	{
2154	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2155	bool const fClear = !fIsNstGstVmcs ? true
2156	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2157	#else
2158	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2159	bool const fClear = true;
2160	#endif
2161	if (fClear)
2162	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2163	}
2164	else
2165	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2166	}
2167
2168
2169	/**
2170	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2171	* area.
2172	*
2173	* @returns VBox status code.
2174	* @param pVCpu The cross context virtual CPU structure.
2175	* @param pVmcsInfo The VMCS info. object.
2176	* @param cMsrs The number of MSRs.
2177	*/
2178	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2179	{
2180	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2181	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2182	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2183	{
2184	/* Commit the MSR counts to the VMCS and update the cache. */
2185	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2186	{
2187	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2188	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
2189	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2190	pVmcsInfo->cEntryMsrLoad = cMsrs;
2191	pVmcsInfo->cExitMsrStore = cMsrs;
2192	pVmcsInfo->cExitMsrLoad = cMsrs;
2193	}
2194	return VINF_SUCCESS;
2195	}
2196
2197	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2198	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2199	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2200	}
2201
2202
2203	/**
2204	* Adds a new (or updates the value of an existing) guest/host MSR
2205	* pair to be swapped during the world-switch as part of the
2206	* auto-load/store MSR area in the VMCS.
2207	*
2208	* @returns VBox status code.
2209	* @param pVCpu The cross context virtual CPU structure.
2210	* @param pVmxTransient The VMX-transient structure.
2211	* @param idMsr The MSR.
2212	* @param uGuestMsrValue Value of the guest MSR.
2213	* @param fSetReadWrite Whether to set the guest read/write access of this
2214	* MSR (thus not causing a VM-exit).
2215	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2216	* necessary.
2217	*/
2218	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2219	bool fSetReadWrite, bool fUpdateHostMsr)
2220	{
2221	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2222	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2223	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2224	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2225	uint32_t i;
2226
2227	/* Paranoia. */
2228	Assert(pGuestMsrLoad);
2229
2230	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2231
2232	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2233	for (i = 0; i < cMsrs; i++)
2234	{
2235	if (pGuestMsrLoad[i].u32Msr == idMsr)
2236	break;
2237	}
2238
2239	bool fAdded = false;
2240	if (i == cMsrs)
2241	{
2242	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2243	++cMsrs;
2244	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2245	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2246
2247	/* Set the guest to read/write this MSR without causing VM-exits. */
2248	if ( fSetReadWrite
2249	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2250	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2251
2252	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2253	fAdded = true;
2254	}
2255
2256	/* Update the MSR value for the newly added or already existing MSR. */
2257	pGuestMsrLoad[i].u32Msr = idMsr;
2258	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2259
2260	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2261	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2262	{
2263	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2264	pGuestMsrStore[i].u32Msr = idMsr;
2265	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2266	}
2267
2268	/* Update the corresponding slot in the host MSR area. */
2269	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2270	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2271	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2272	pHostMsr[i].u32Msr = idMsr;
2273
2274	/*
2275	* Only if the caller requests to update the host MSR value AND we've newly added the
2276	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2277	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2278	*
2279	* We do this for performance reasons since reading MSRs may be quite expensive.
2280	*/
2281	if (fAdded)
2282	{
2283	if (fUpdateHostMsr)
2284	{
2285	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2286	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2287	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2288	}
2289	else
2290	{
2291	/* Someone else can do the work. */
2292	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2293	}
2294	}
2295	return VINF_SUCCESS;
2296	}
2297
2298
2299	/**
2300	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2301	* auto-load/store MSR area in the VMCS.
2302	*
2303	* @returns VBox status code.
2304	* @param pVCpu The cross context virtual CPU structure.
2305	* @param pVmxTransient The VMX-transient structure.
2306	* @param idMsr The MSR.
2307	*/
2308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2309	{
2310	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2311	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2312	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2313	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2314
2315	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2316
2317	for (uint32_t i = 0; i < cMsrs; i++)
2318	{
2319	/* Find the MSR. */
2320	if (pGuestMsrLoad[i].u32Msr == idMsr)
2321	{
2322	/*
2323	* If it's the last MSR, we only need to reduce the MSR count.
2324	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2325	*/
2326	if (i < cMsrs - 1)
2327	{
2328	/* Remove it from the VM-entry MSR-load area. */
2329	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2330	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2331
2332	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2333	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2334	{
2335	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2336	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2337	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2338	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2339	}
2340
2341	/* Remove it from the VM-exit MSR-load area. */
2342	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2343	Assert(pHostMsr[i].u32Msr == idMsr);
2344	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2345	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2346	}
2347
2348	/* Reduce the count to reflect the removed MSR and bail. */
2349	--cMsrs;
2350	break;
2351	}
2352	}
2353
2354	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2355	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2356	{
2357	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2358	AssertRCReturn(rc, rc);
2359
2360	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2361	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2362	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2363
2364	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2365	return VINF_SUCCESS;
2366	}
2367
2368	return VERR_NOT_FOUND;
2369	}
2370
2371
2372	/**
2373	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2374	*
2375	* @returns @c true if found, @c false otherwise.
2376	* @param pVmcsInfo The VMCS info. object.
2377	* @param idMsr The MSR to find.
2378	*/
2379	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2380	{
2381	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2382	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2383	Assert(pMsrs);
2384	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2385	for (uint32_t i = 0; i < cMsrs; i++)
2386	{
2387	if (pMsrs[i].u32Msr == idMsr)
2388	return true;
2389	}
2390	return false;
2391	}
2392
2393
2394	/**
2395	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2396	*
2397	* @param pVCpu The cross context virtual CPU structure.
2398	* @param pVmcsInfo The VMCS info. object.
2399	*
2400	* @remarks No-long-jump zone!!!
2401	*/
2402	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2403	{
2404	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2405
2406	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2407	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2408	Assert(pHostMsrLoad);
2409	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2410	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2411	for (uint32_t i = 0; i < cMsrs; i++)
2412	{
2413	/*
2414	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2415	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2416	*/
2417	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2418	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2419	else
2420	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2421	}
2422	}
2423
2424
2425	/**
2426	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2427	* perform lazy restoration of the host MSRs while leaving VT-x.
2428	*
2429	* @param pVCpu The cross context virtual CPU structure.
2430	*
2431	* @remarks No-long-jump zone!!!
2432	*/
2433	static void hmR0VmxLazySaveHostMsrs(PVMCPUCC pVCpu)
2434	{
2435	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2436
2437	/*
2438	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2439	*/
2440	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2441	{
2442	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2443	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2444	{
2445	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2446	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2447	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2448	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2449	}
2450	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2451	}
2452	}
2453
2454
2455	/**
2456	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2457	* lazily while leaving VT-x.
2458	*
2459	* @returns true if it does, false otherwise.
2460	* @param pVCpu The cross context virtual CPU structure.
2461	* @param idMsr The MSR to check.
2462	*/
2463	static bool hmR0VmxIsLazyGuestMsr(PCVMCPUCC pVCpu, uint32_t idMsr)
2464	{
2465	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2466	{
2467	switch (idMsr)
2468	{
2469	case MSR_K8_LSTAR:
2470	case MSR_K6_STAR:
2471	case MSR_K8_SF_MASK:
2472	case MSR_K8_KERNEL_GS_BASE:
2473	return true;
2474	}
2475	}
2476	return false;
2477	}
2478
2479
2480	/**
2481	* Loads a set of guests MSRs to allow read/passthru to the guest.
2482	*
2483	* The name of this function is slightly confusing. This function does NOT
2484	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2485	* common prefix for functions dealing with "lazy restoration" of the shared
2486	* MSRs.
2487	*
2488	* @param pVCpu The cross context virtual CPU structure.
2489	*
2490	* @remarks No-long-jump zone!!!
2491	*/
2492	static void hmR0VmxLazyLoadGuestMsrs(PVMCPUCC pVCpu)
2493	{
2494	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2495	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2496
2497	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2498	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2499	{
2500	/*
2501	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2502	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2503	* we can skip a few MSR writes.
2504	*
2505	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2506	* guest MSR values in the guest-CPU context might be different to what's currently
2507	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2508	* CPU, see @bugref{8728}.
2509	*/
2510	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2511	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2512	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2513	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2514	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2515	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2516	{
2517	#ifdef VBOX_STRICT
2518	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2519	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2520	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2521	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2522	#endif
2523	}
2524	else
2525	{
2526	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2527	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2528	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2529	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2530	}
2531	}
2532	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2533	}
2534
2535
2536	/**
2537	* Performs lazy restoration of the set of host MSRs if they were previously
2538	* loaded with guest MSR values.
2539	*
2540	* @param pVCpu The cross context virtual CPU structure.
2541	*
2542	* @remarks No-long-jump zone!!!
2543	* @remarks The guest MSRs should have been saved back into the guest-CPU
2544	* context by hmR0VmxImportGuestState()!!!
2545	*/
2546	static void hmR0VmxLazyRestoreHostMsrs(PVMCPUCC pVCpu)
2547	{
2548	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2549	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2550
2551	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2552	{
2553	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2554	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2555	{
2556	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2557	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2558	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2559	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2560	}
2561	}
2562	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2563	}
2564
2565
2566	/**
2567	* Verifies that our cached values of the VMCS fields are all consistent with
2568	* what's actually present in the VMCS.
2569	*
2570	* @returns VBox status code.
2571	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2572	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2573	* VMCS content. HMCPU error-field is
2574	* updated, see VMX_VCI_XXX.
2575	* @param pVCpu The cross context virtual CPU structure.
2576	* @param pVmcsInfo The VMCS info. object.
2577	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2578	*/
2579	static int hmR0VmxCheckVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2580	{
2581	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2582
2583	uint32_t u32Val;
2584	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2585	AssertRC(rc);
2586	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2587	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2588	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2589	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2590
2591	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2592	AssertRC(rc);
2593	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2594	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2595	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2596	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2597
2598	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2599	AssertRC(rc);
2600	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2601	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2602	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2603	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2604
2605	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2606	AssertRC(rc);
2607	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2608	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2609	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2610	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2611
2612	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2613	{
2614	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2615	AssertRC(rc);
2616	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2617	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2618	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2619	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2620	}
2621
2622	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2623	AssertRC(rc);
2624	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2625	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2626	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2627	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2628
2629	uint64_t u64Val;
2630	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2631	AssertRC(rc);
2632	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2633	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2634	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2635	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2636
2637	NOREF(pcszVmcs);
2638	return VINF_SUCCESS;
2639	}
2640
2641
2642	#ifdef VBOX_STRICT
2643	/**
2644	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2645	*
2646	* @param pVCpu The cross context virtual CPU structure.
2647	* @param pVmcsInfo The VMCS info. object.
2648	*/
2649	static void hmR0VmxCheckHostEferMsr(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2650	{
2651	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2652
2653	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2654	{
2655	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2656	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2657	uint64_t uVmcsEferMsrVmcs;
2658	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2659	AssertRC(rc);
2660
2661	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2662	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2663	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2664	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2665	}
2666	}
2667
2668
2669	/**
2670	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2671	* VMCS are correct.
2672	*
2673	* @param pVCpu The cross context virtual CPU structure.
2674	* @param pVmcsInfo The VMCS info. object.
2675	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2676	*/
2677	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2678	{
2679	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2680
2681	/* Read the various MSR-area counts from the VMCS. */
2682	uint32_t cEntryLoadMsrs;
2683	uint32_t cExitStoreMsrs;
2684	uint32_t cExitLoadMsrs;
2685	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2686	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2687	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2688
2689	/* Verify all the MSR counts are the same. */
2690	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2691	Assert(cExitStoreMsrs == cExitLoadMsrs);
2692	uint32_t const cMsrs = cExitLoadMsrs;
2693
2694	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2695	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2696
2697	/* Verify the MSR counts are within the allocated page size. */
2698	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2699
2700	/* Verify the relevant contents of the MSR areas match. */
2701	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2702	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2703	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2704	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2705	for (uint32_t i = 0; i < cMsrs; i++)
2706	{
2707	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2708	if (fSeparateExitMsrStorePage)
2709	{
2710	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2711	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2712	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2713	}
2714
2715	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2716	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2717	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2718
2719	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2720	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2721	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2722	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2723
2724	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2725	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2726	if (fIsEferMsr)
2727	{
2728	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2729	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2730	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2731	}
2732
2733	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2734	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2735	{
2736	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2737	if (fIsEferMsr)
2738	{
2739	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2740	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2741	}
2742	else
2743	{
2744	if (!fIsNstGstVmcs)
2745	{
2746	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2747	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2748	}
2749	else
2750	{
2751	/*
2752	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2753	* execute a nested-guest with MSR passthrough.
2754	*
2755	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2756	* allow passthrough too.
2757	*/
2758	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2759	Assert(pvMsrBitmapNstGst);
2760	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2761	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2762	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2763	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2764	}
2765	}
2766	}
2767
2768	/* Move to the next MSR. */
2769	pHostMsrLoad++;
2770	pGuestMsrLoad++;
2771	pGuestMsrStore++;
2772	}
2773	}
2774	#endif /* VBOX_STRICT */
2775
2776
2777	/**
2778	* Flushes the TLB using EPT.
2779	*
2780	* @returns VBox status code.
2781	* @param pVCpu The cross context virtual CPU structure of the calling
2782	* EMT. Can be NULL depending on @a enmTlbFlush.
2783	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2784	* enmTlbFlush.
2785	* @param enmTlbFlush Type of flush.
2786	*
2787	* @remarks Caller is responsible for making sure this function is called only
2788	* when NestedPaging is supported and providing @a enmTlbFlush that is
2789	* supported by the CPU.
2790	* @remarks Can be called with interrupts disabled.
2791	*/
2792	static void hmR0VmxFlushEpt(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2793	{
2794	uint64_t au64Descriptor[2];
2795	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2796	au64Descriptor[0] = 0;
2797	else
2798	{
2799	Assert(pVCpu);
2800	Assert(pVmcsInfo);
2801	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2802	}
2803	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2804
2805	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2806	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2807
2808	if ( RT_SUCCESS(rc)
2809	&& pVCpu)
2810	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2811	}
2812
2813
2814	/**
2815	* Flushes the TLB using VPID.
2816	*
2817	* @returns VBox status code.
2818	* @param pVCpu The cross context virtual CPU structure of the calling
2819	* EMT. Can be NULL depending on @a enmTlbFlush.
2820	* @param enmTlbFlush Type of flush.
2821	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2822	* on @a enmTlbFlush).
2823	*
2824	* @remarks Can be called with interrupts disabled.
2825	*/
2826	static void hmR0VmxFlushVpid(PVMCPUCC pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2827	{
2828	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2829
2830	uint64_t au64Descriptor[2];
2831	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2832	{
2833	au64Descriptor[0] = 0;
2834	au64Descriptor[1] = 0;
2835	}
2836	else
2837	{
2838	AssertPtr(pVCpu);
2839	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2840	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2841	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2842	au64Descriptor[1] = GCPtr;
2843	}
2844
2845	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2846	AssertMsg(rc == VINF_SUCCESS,
2847	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2848
2849	if ( RT_SUCCESS(rc)
2850	&& pVCpu)
2851	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2852	NOREF(rc);
2853	}
2854
2855
2856	/**
2857	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2858	* otherwise there is nothing really to invalidate.
2859	*
2860	* @returns VBox status code.
2861	* @param pVCpu The cross context virtual CPU structure.
2862	* @param GCVirt Guest virtual address of the page to invalidate.
2863	*/
2864	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
2865	{
2866	AssertPtr(pVCpu);
2867	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2868
2869	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2870	{
2871	/*
2872	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2873	* the EPT case. See @bugref{6043} and @bugref{6177}.
2874	*
2875	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2876	* as this function maybe called in a loop with individual addresses.
2877	*/
2878	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2879	if (pVM->hm.s.vmx.fVpid)
2880	{
2881	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2882	if (fVpidFlush)
2883	{
2884	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2885	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2886	}
2887	else
2888	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2889	}
2890	else if (pVM->hm.s.fNestedPaging)
2891	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2892	}
2893
2894	return VINF_SUCCESS;
2895	}
2896
2897
2898	/**
2899	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2900	* case where neither EPT nor VPID is supported by the CPU.
2901	*
2902	* @param pHostCpu The HM physical-CPU structure.
2903	* @param pVCpu The cross context virtual CPU structure.
2904	*
2905	* @remarks Called with interrupts disabled.
2906	*/
2907	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2908	{
2909	AssertPtr(pVCpu);
2910	AssertPtr(pHostCpu);
2911
2912	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2913
2914	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2915	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2916	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2917	pVCpu->hm.s.fForceTLBFlush = false;
2918	return;
2919	}
2920
2921
2922	/**
2923	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2924	*
2925	* @param pHostCpu The HM physical-CPU structure.
2926	* @param pVCpu The cross context virtual CPU structure.
2927	* @param pVmcsInfo The VMCS info. object.
2928	*
2929	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2930	* nomenclature. The reason is, to avoid confusion in compare statements
2931	* since the host-CPU copies are named "ASID".
2932	*
2933	* @remarks Called with interrupts disabled.
2934	*/
2935	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2936	{
2937	#ifdef VBOX_WITH_STATISTICS
2938	bool fTlbFlushed = false;
2939	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2940	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2941	if (!fTlbFlushed) \
2942	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2943	} while (0)
2944	#else
2945	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2946	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2947	#endif
2948
2949	AssertPtr(pVCpu);
2950	AssertPtr(pHostCpu);
2951	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2952
2953	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2954	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2955	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2956	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2957
2958	/*
2959	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2960	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2961	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2962	* cannot reuse the current ASID anymore.
2963	*/
2964	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2965	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2966	{
2967	++pHostCpu->uCurrentAsid;
2968	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2969	{
2970	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2971	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2972	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2973	}
2974
2975	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2976	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2977	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2978
2979	/*
2980	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2981	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2982	*/
2983	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2984	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2985	HMVMX_SET_TAGGED_TLB_FLUSHED();
2986	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2987	}
2988	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2989	{
2990	/*
2991	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2992	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2993	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2994	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2995	* mappings, see @bugref{6568}.
2996	*
2997	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2998	*/
2999	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3000	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3001	HMVMX_SET_TAGGED_TLB_FLUSHED();
3002	}
3003	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3004	{
3005	/*
3006	* The nested-guest specifies its own guest-physical address to use as the APIC-access
3007	* address which requires flushing the TLB of EPT cached structures.
3008	*
3009	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
3010	*/
3011	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3012	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3013	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3014	HMVMX_SET_TAGGED_TLB_FLUSHED();
3015	}
3016
3017
3018	pVCpu->hm.s.fForceTLBFlush = false;
3019	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3020
3021	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3022	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3023	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3024	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3025	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3026	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3027	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3028	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3029	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3030
3031	/* Update VMCS with the VPID. */
3032	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3033	AssertRC(rc);
3034
3035	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3036	}
3037
3038
3039	/**
3040	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3041	*
3042	* @param pHostCpu The HM physical-CPU structure.
3043	* @param pVCpu The cross context virtual CPU structure.
3044	* @param pVmcsInfo The VMCS info. object.
3045	*
3046	* @remarks Called with interrupts disabled.
3047	*/
3048	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3049	{
3050	AssertPtr(pVCpu);
3051	AssertPtr(pHostCpu);
3052	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3053	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3054	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3055
3056	/*
3057	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3058	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3059	*/
3060	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3061	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3062	{
3063	pVCpu->hm.s.fForceTLBFlush = true;
3064	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3065	}
3066
3067	/* Check for explicit TLB flushes. */
3068	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3069	{
3070	pVCpu->hm.s.fForceTLBFlush = true;
3071	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3072	}
3073
3074	/* Check for TLB flushes while switching to/from a nested-guest. */
3075	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3076	{
3077	pVCpu->hm.s.fForceTLBFlush = true;
3078	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3079	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3080	}
3081
3082	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3083	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3084
3085	if (pVCpu->hm.s.fForceTLBFlush)
3086	{
3087	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3088	pVCpu->hm.s.fForceTLBFlush = false;
3089	}
3090	}
3091
3092
3093	/**
3094	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3095	*
3096	* @param pHostCpu The HM physical-CPU structure.
3097	* @param pVCpu The cross context virtual CPU structure.
3098	*
3099	* @remarks Called with interrupts disabled.
3100	*/
3101	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
3102	{
3103	AssertPtr(pVCpu);
3104	AssertPtr(pHostCpu);
3105	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3106	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3107	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3108
3109	/*
3110	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3111	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3112	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3113	* cannot reuse the current ASID anymore.
3114	*/
3115	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3116	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3117	{
3118	pVCpu->hm.s.fForceTLBFlush = true;
3119	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3120	}
3121
3122	/* Check for explicit TLB flushes. */
3123	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3124	{
3125	/*
3126	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3127	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3128	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3129	* include fExplicitFlush's too) - an obscure corner case.
3130	*/
3131	pVCpu->hm.s.fForceTLBFlush = true;
3132	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3133	}
3134
3135	/* Check for TLB flushes while switching to/from a nested-guest. */
3136	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3137	{
3138	pVCpu->hm.s.fForceTLBFlush = true;
3139	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3140	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3141	}
3142
3143	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3144	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3145	if (pVCpu->hm.s.fForceTLBFlush)
3146	{
3147	++pHostCpu->uCurrentAsid;
3148	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3149	{
3150	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3151	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3152	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3153	}
3154
3155	pVCpu->hm.s.fForceTLBFlush = false;
3156	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3157	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3158	if (pHostCpu->fFlushAsidBeforeUse)
3159	{
3160	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3161	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3162	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3163	{
3164	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3165	pHostCpu->fFlushAsidBeforeUse = false;
3166	}
3167	else
3168	{
3169	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3170	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3171	}
3172	}
3173	}
3174
3175	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3176	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3177	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3178	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3179	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3180	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3181	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3182
3183	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3184	AssertRC(rc);
3185	}
3186
3187
3188	/**
3189	* Flushes the guest TLB entry based on CPU capabilities.
3190	*
3191	* @param pHostCpu The HM physical-CPU structure.
3192	* @param pVCpu The cross context virtual CPU structure.
3193	* @param pVmcsInfo The VMCS info. object.
3194	*
3195	* @remarks Called with interrupts disabled.
3196	*/
3197	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3198	{
3199	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3200	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3201	#endif
3202	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3203	switch (pVM->hm.s.vmx.enmTlbFlushType)
3204	{
3205	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3206	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3207	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3208	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3209	default:
3210	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3211	break;
3212	}
3213	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3214	}
3215
3216
3217	/**
3218	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3219	* TLB entries from the host TLB before VM-entry.
3220	*
3221	* @returns VBox status code.
3222	* @param pVM The cross context VM structure.
3223	*/
3224	static int hmR0VmxSetupTaggedTlb(PVMCC pVM)
3225	{
3226	/*
3227	* Determine optimal flush type for nested paging.
3228	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3229	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3230	*/
3231	if (pVM->hm.s.fNestedPaging)
3232	{
3233	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3234	{
3235	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3236	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3237	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3238	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3239	else
3240	{
3241	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3242	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3243	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3244	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3245	}
3246
3247	/* Make sure the write-back cacheable memory type for EPT is supported. */
3248	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3249	{
3250	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3251	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3252	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3253	}
3254
3255	/* EPT requires a page-walk length of 4. */
3256	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3257	{
3258	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3259	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3260	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3261	}
3262	}
3263	else
3264	{
3265	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3266	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3267	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3268	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3269	}
3270	}
3271
3272	/*
3273	* Determine optimal flush type for VPID.
3274	*/
3275	if (pVM->hm.s.vmx.fVpid)
3276	{
3277	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3278	{
3279	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3280	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3281	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3282	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3283	else
3284	{
3285	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3286	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3287	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3288	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3289	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3290	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3291	pVM->hm.s.vmx.fVpid = false;
3292	}
3293	}
3294	else
3295	{
3296	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3297	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3298	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3299	pVM->hm.s.vmx.fVpid = false;
3300	}
3301	}
3302
3303	/*
3304	* Setup the handler for flushing tagged-TLBs.
3305	*/
3306	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3307	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3308	else if (pVM->hm.s.fNestedPaging)
3309	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3310	else if (pVM->hm.s.vmx.fVpid)
3311	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3312	else
3313	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3314	return VINF_SUCCESS;
3315	}
3316
3317
3318	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3319	/**
3320	* Sets up the shadow VMCS fields arrays.
3321	*
3322	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3323	* executing the guest.
3324	*
3325	* @returns VBox status code.
3326	* @param pVM The cross context VM structure.
3327	*/
3328	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVMCC pVM)
3329	{
3330	/*
3331	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3332	* when the host does not support it.
3333	*/
3334	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3335	if ( !fGstVmwriteAll
3336	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3337	{ /* likely. */ }
3338	else
3339	{
3340	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3341	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3342	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3343	}
3344
3345	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3346	uint32_t cRwFields = 0;
3347	uint32_t cRoFields = 0;
3348	for (uint32_t i = 0; i < cVmcsFields; i++)
3349	{
3350	VMXVMCSFIELD VmcsField;
3351	VmcsField.u = g_aVmcsFields[i];
3352
3353	/*
3354	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3355	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3356	* in the shadow VMCS fields array as they would be redundant.
3357	*
3358	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3359	* we must not include it in the shadow VMCS fields array. Guests attempting to
3360	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3361	* the required behavior.
3362	*/
3363	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3364	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3365	{
3366	/*
3367	* Read-only fields are placed in a separate array so that while syncing shadow
3368	* VMCS fields later (which is more performance critical) we can avoid branches.
3369	*
3370	* However, if the guest can write to all fields (including read-only fields),
3371	* we treat it a as read/write field. Otherwise, writing to these fields would
3372	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3373	*/
3374	if ( fGstVmwriteAll
3375	\|\| !VMXIsVmcsFieldReadOnly(VmcsField.u))
3376	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3377	else
3378	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3379	}
3380	}
3381
3382	/* Update the counts. */
3383	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3384	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3385	return VINF_SUCCESS;
3386	}
3387
3388
3389	/**
3390	* Sets up the VMREAD and VMWRITE bitmaps.
3391	*
3392	* @param pVM The cross context VM structure.
3393	*/
3394	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVMCC pVM)
3395	{
3396	/*
3397	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3398	*/
3399	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3400	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3401	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3402	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3403	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3404
3405	/*
3406	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3407	* VMREAD and VMWRITE bitmaps.
3408	*/
3409	{
3410	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3411	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3412	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3413	{
3414	uint32_t const uVmcsField = paShadowVmcsFields[i];
3415	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3416	Assert(uVmcsField >> 3 < cbBitmap);
3417	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3418	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3419	}
3420	}
3421
3422	/*
3423	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3424	* if the host supports VMWRITE to all supported VMCS fields.
3425	*/
3426	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3427	{
3428	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3429	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3430	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3431	{
3432	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3433	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3434	Assert(uVmcsField >> 3 < cbBitmap);
3435	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3436	}
3437	}
3438	}
3439	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3440
3441
3442	/**
3443	* Sets up the virtual-APIC page address for the VMCS.
3444	*
3445	* @param pVmcsInfo The VMCS info. object.
3446	*/
3447	DECLINLINE(void) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
3448	{
3449	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3450	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3451	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3452	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3453	AssertRC(rc);
3454	}
3455
3456
3457	/**
3458	* Sets up the MSR-bitmap address for the VMCS.
3459	*
3460	* @param pVmcsInfo The VMCS info. object.
3461	*/
3462	DECLINLINE(void) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
3463	{
3464	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3465	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3466	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3467	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3468	AssertRC(rc);
3469	}
3470
3471
3472	/**
3473	* Sets up the APIC-access page address for the VMCS.
3474	*
3475	* @param pVCpu The cross context virtual CPU structure.
3476	*/
3477	DECLINLINE(void) hmR0VmxSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
3478	{
3479	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3480	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3481	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3482	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3483	AssertRC(rc);
3484	}
3485
3486
3487	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3488	/**
3489	* Sets up the VMREAD bitmap address for the VMCS.
3490	*
3491	* @param pVCpu The cross context virtual CPU structure.
3492	*/
3493	DECLINLINE(void) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
3494	{
3495	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3496	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3497	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3498	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3499	AssertRC(rc);
3500	}
3501
3502
3503	/**
3504	* Sets up the VMWRITE bitmap address for the VMCS.
3505	*
3506	* @param pVCpu The cross context virtual CPU structure.
3507	*/
3508	DECLINLINE(void) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
3509	{
3510	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3511	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3512	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3513	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3514	AssertRC(rc);
3515	}
3516	#endif
3517
3518
3519	/**
3520	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3521	* in the VMCS.
3522	*
3523	* @returns VBox status code.
3524	* @param pVmcsInfo The VMCS info. object.
3525	*/
3526	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
3527	{
3528	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3529	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3530	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3531
3532	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3533	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3534	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3535
3536	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3537	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3538	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3539
3540	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad); AssertRC(rc);
3541	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore); AssertRC(rc);
3542	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad); AssertRC(rc);
3543	return VINF_SUCCESS;
3544	}
3545
3546
3547	/**
3548	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3549	*
3550	* @param pVCpu The cross context virtual CPU structure.
3551	* @param pVmcsInfo The VMCS info. object.
3552	*/
3553	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3554	{
3555	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3556
3557	/*
3558	* The guest can access the following MSRs (read, write) without causing
3559	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3560	*/
3561	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3562	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3563	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3564	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3565	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3566	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3567
3568	/*
3569	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3570	* associated with then. We never need to intercept access (writes need to be
3571	* executed without causing a VM-exit, reads will #GP fault anyway).
3572	*
3573	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3574	* read/write them. We swap the the guest/host MSR value using the
3575	* auto-load/store MSR area.
3576	*/
3577	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3578	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3579	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3580	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3581	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3582	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3583
3584	/*
3585	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3586	* required for 64-bit guests.
3587	*/
3588	if (pVM->hm.s.fAllow64BitGuests)
3589	{
3590	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3591	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3592	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3593	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3594	}
3595
3596	/*
3597	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3598	*/
3599	#ifdef VBOX_STRICT
3600	Assert(pVmcsInfo->pvMsrBitmap);
3601	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3602	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3603	#endif
3604	}
3605
3606
3607	/**
3608	* Sets up pin-based VM-execution controls in the VMCS.
3609	*
3610	* @returns VBox status code.
3611	* @param pVCpu The cross context virtual CPU structure.
3612	* @param pVmcsInfo The VMCS info. object.
3613	*/
3614	static int hmR0VmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3615	{
3616	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3617	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3618	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3619
3620	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3621	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3622
3623	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3624	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3625
3626	/* Enable the VMX-preemption timer. */
3627	if (pVM->hm.s.vmx.fUsePreemptTimer)
3628	{
3629	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3630	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3631	}
3632
3633	#if 0
3634	/* Enable posted-interrupt processing. */
3635	if (pVM->hm.s.fPostedIntrs)
3636	{
3637	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3638	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3639	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3640	}
3641	#endif
3642
3643	if ((fVal & fZap) != fVal)
3644	{
3645	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3646	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3647	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3648	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3649	}
3650
3651	/* Commit it to the VMCS and update our cache. */
3652	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3653	AssertRC(rc);
3654	pVmcsInfo->u32PinCtls = fVal;
3655
3656	return VINF_SUCCESS;
3657	}
3658
3659
3660	/**
3661	* Sets up secondary processor-based VM-execution controls in the VMCS.
3662	*
3663	* @returns VBox status code.
3664	* @param pVCpu The cross context virtual CPU structure.
3665	* @param pVmcsInfo The VMCS info. object.
3666	*/
3667	static int hmR0VmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3668	{
3669	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3670	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3671	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3672
3673	/* WBINVD causes a VM-exit. */
3674	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3675	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3676
3677	/* Enable EPT (aka nested-paging). */
3678	if (pVM->hm.s.fNestedPaging)
3679	fVal \|= VMX_PROC_CTLS2_EPT;
3680
3681	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3682	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3683	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3684	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3685	fVal \|= VMX_PROC_CTLS2_INVPCID;
3686
3687	/* Enable VPID. */
3688	if (pVM->hm.s.vmx.fVpid)
3689	fVal \|= VMX_PROC_CTLS2_VPID;
3690
3691	/* Enable unrestricted guest execution. */
3692	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3693	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3694
3695	#if 0
3696	if (pVM->hm.s.fVirtApicRegs)
3697	{
3698	/* Enable APIC-register virtualization. */
3699	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3700	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3701
3702	/* Enable virtual-interrupt delivery. */
3703	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3704	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3705	}
3706	#endif
3707
3708	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3709	where the TPR shadow resides. */
3710	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3711	* done dynamically. */
3712	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3713	{
3714	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3715	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3716	}
3717
3718	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3719	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3720	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3721	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3722	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3723
3724	/* Enable Pause-Loop exiting. */
3725	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3726	&& pVM->hm.s.vmx.cPleGapTicks
3727	&& pVM->hm.s.vmx.cPleWindowTicks)
3728	{
3729	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3730
3731	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
3732	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
3733	}
3734
3735	if ((fVal & fZap) != fVal)
3736	{
3737	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3738	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3739	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3740	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3741	}
3742
3743	/* Commit it to the VMCS and update our cache. */
3744	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3745	AssertRC(rc);
3746	pVmcsInfo->u32ProcCtls2 = fVal;
3747
3748	return VINF_SUCCESS;
3749	}
3750
3751
3752	/**
3753	* Sets up processor-based VM-execution controls in the VMCS.
3754	*
3755	* @returns VBox status code.
3756	* @param pVCpu The cross context virtual CPU structure.
3757	* @param pVmcsInfo The VMCS info. object.
3758	*/
3759	static int hmR0VmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3760	{
3761	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3762
3763	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3764	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3765
3766	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3767	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3768	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3769	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3770	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3771	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3772	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3773
3774	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3775	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3776	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3777	{
3778	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3779	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3780	}
3781
3782	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3783	if (!pVM->hm.s.fNestedPaging)
3784	{
3785	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3786	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3787	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3788	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3789	}
3790
3791	/* Use TPR shadowing if supported by the CPU. */
3792	if ( PDMHasApic(pVM)
3793	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3794	{
3795	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3796	/* CR8 writes cause a VM-exit based on TPR threshold. */
3797	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3798	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3799	hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3800	}
3801	else
3802	{
3803	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3804	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3805	if (pVM->hm.s.fAllow64BitGuests)
3806	{
3807	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3808	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3809	}
3810	}
3811
3812	/* Use MSR-bitmaps if supported by the CPU. */
3813	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3814	{
3815	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3816	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3817	}
3818
3819	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3820	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3821	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3822
3823	if ((fVal & fZap) != fVal)
3824	{
3825	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3826	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3827	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3828	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3829	}
3830
3831	/* Commit it to the VMCS and update our cache. */
3832	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3833	AssertRC(rc);
3834	pVmcsInfo->u32ProcCtls = fVal;
3835
3836	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3837	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3838	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3839
3840	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3841	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3842	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3843
3844	/* Sanity check, should not really happen. */
3845	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3846	{ /* likely */ }
3847	else
3848	{
3849	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3850	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3851	}
3852
3853	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3854	return VINF_SUCCESS;
3855	}
3856
3857
3858	/**
3859	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3860	* Processor-based VM-execution) control fields in the VMCS.
3861	*
3862	* @returns VBox status code.
3863	* @param pVCpu The cross context virtual CPU structure.
3864	* @param pVmcsInfo The VMCS info. object.
3865	*/
3866	static int hmR0VmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3867	{
3868	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3869	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3870	{
3871	hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3872	hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3873	}
3874	#endif
3875
3876	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3877	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3878	AssertRC(rc);
3879
3880	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3881	if (RT_SUCCESS(rc))
3882	{
3883	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3884	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3885
3886	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
3887	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
3888
3889	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3890	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3891	return VINF_SUCCESS;
3892	}
3893	else
3894	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3895	return rc;
3896	}
3897
3898
3899	/**
3900	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3901	*
3902	* We shall setup those exception intercepts that don't change during the
3903	* lifetime of the VM here. The rest are done dynamically while loading the
3904	* guest state.
3905	*
3906	* @param pVCpu The cross context virtual CPU structure.
3907	* @param pVmcsInfo The VMCS info. object.
3908	*/
3909	static void hmR0VmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3910	{
3911	/*
3912	* The following exceptions are always intercepted:
3913	*
3914	* #AC - To prevent the guest from hanging the CPU.
3915	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3916	* recursive #DBs can cause a CPU hang.
3917	* #PF - To sync our shadow page tables when nested-paging is not used.
3918	*/
3919	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3920	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3921	\| RT_BIT(X86_XCPT_DB)
3922	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3923
3924	/* Commit it to the VMCS. */
3925	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3926	AssertRC(rc);
3927
3928	/* Update our cache of the exception bitmap. */
3929	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3930	}
3931
3932
3933	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3934	/**
3935	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3936	*
3937	* @returns VBox status code.
3938	* @param pVCpu The cross context virtual CPU structure.
3939	* @param pVmcsInfo The VMCS info. object.
3940	*/
3941	static int hmR0VmxSetupVmcsCtlsNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3942	{
3943	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3944	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3945	AssertRC(rc);
3946
3947	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3948	if (RT_SUCCESS(rc))
3949	{
3950	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3951	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3952
3953	/* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
3954	Assert(!pVmcsInfo->u64Cr0Mask);
3955	Assert(!pVmcsInfo->u64Cr4Mask);
3956	return VINF_SUCCESS;
3957	}
3958	else
3959	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3960	return rc;
3961	}
3962	#endif
3963
3964
3965	/**
3966	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3967	* VMX.
3968	*
3969	* @returns VBox status code.
3970	* @param pVCpu The cross context virtual CPU structure.
3971	* @param pVmcsInfo The VMCS info. object.
3972	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3973	*/
3974	static int hmR0VmxSetupVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3975	{
3976	Assert(pVmcsInfo->pvVmcs);
3977	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3978
3979	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3980	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3981	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3982	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3983
3984	LogFlowFunc(("\n"));
3985
3986	/*
3987	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3988	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3989	*/
3990	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3991	if (RT_SUCCESS(rc))
3992	{
3993	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3994	if (RT_SUCCESS(rc))
3995	{
3996	if (!fIsNstGstVmcs)
3997	{
3998	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3999	if (RT_SUCCESS(rc))
4000	{
4001	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4002	if (RT_SUCCESS(rc))
4003	{
4004	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4005	if (RT_SUCCESS(rc))
4006	{
4007	hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4008	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4009	/*
4010	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4011	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4012	* making it fit for use when VMCS shadowing is later enabled.
4013	*/
4014	if (pVmcsInfo->pvShadowVmcs)
4015	{
4016	VMXVMCSREVID VmcsRevId;
4017	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4018	VmcsRevId.n.fIsShadowVmcs = 1;
4019	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4020	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4021	if (RT_SUCCESS(rc))
4022	{ /* likely */ }
4023	else
4024	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4025	}
4026	#endif
4027	}
4028	else
4029	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4030	}
4031	else
4032	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4033	}
4034	else
4035	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4036	}
4037	else
4038	{
4039	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4040	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4041	if (RT_SUCCESS(rc))
4042	{ /* likely */ }
4043	else
4044	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4045	#else
4046	AssertFailed();
4047	#endif
4048	}
4049	}
4050	else
4051	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4052	}
4053	else
4054	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4055
4056	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4057	if (RT_SUCCESS(rc))
4058	{
4059	rc = hmR0VmxClearVmcs(pVmcsInfo);
4060	if (RT_SUCCESS(rc))
4061	{ /* likely */ }
4062	else
4063	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4064	}
4065
4066	/*
4067	* Update the last-error record both for failures and success, so we
4068	* can propagate the status code back to ring-3 for diagnostics.
4069	*/
4070	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4071	NOREF(pszVmcs);
4072	return rc;
4073	}
4074
4075
4076	/**
4077	* Does global VT-x initialization (called during module initialization).
4078	*
4079	* @returns VBox status code.
4080	*/
4081	VMMR0DECL(int) VMXR0GlobalInit(void)
4082	{
4083	#ifdef HMVMX_USE_FUNCTION_TABLE
4084	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4085	# ifdef VBOX_STRICT
4086	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4087	Assert(g_apfnVMExitHandlers[i]);
4088	# endif
4089	#endif
4090	return VINF_SUCCESS;
4091	}
4092
4093
4094	/**
4095	* Does global VT-x termination (called during module termination).
4096	*/
4097	VMMR0DECL(void) VMXR0GlobalTerm()
4098	{
4099	/* Nothing to do currently. */
4100	}
4101
4102
4103	/**
4104	* Sets up and activates VT-x on the current CPU.
4105	*
4106	* @returns VBox status code.
4107	* @param pHostCpu The HM physical-CPU structure.
4108	* @param pVM The cross context VM structure. Can be
4109	* NULL after a host resume operation.
4110	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4111	* fEnabledByHost is @c true).
4112	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4113	* @a fEnabledByHost is @c true).
4114	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4115	* enable VT-x on the host.
4116	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4117	*/
4118	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4119	PCSUPHWVIRTMSRS pHwvirtMsrs)
4120	{
4121	AssertPtr(pHostCpu);
4122	AssertPtr(pHwvirtMsrs);
4123	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4124
4125	/* Enable VT-x if it's not already enabled by the host. */
4126	if (!fEnabledByHost)
4127	{
4128	int rc = hmR0VmxEnterRootMode(pHostCpu, pVM, HCPhysCpuPage, pvCpuPage);
4129	if (RT_FAILURE(rc))
4130	return rc;
4131	}
4132
4133	/*
4134	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4135	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4136	* invalidated when flushing by VPID.
4137	*/
4138	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4139	{
4140	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4141	pHostCpu->fFlushAsidBeforeUse = false;
4142	}
4143	else
4144	pHostCpu->fFlushAsidBeforeUse = true;
4145
4146	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4147	++pHostCpu->cTlbFlushes;
4148
4149	return VINF_SUCCESS;
4150	}
4151
4152
4153	/**
4154	* Deactivates VT-x on the current CPU.
4155	*
4156	* @returns VBox status code.
4157	* @param pHostCpu The HM physical-CPU structure.
4158	* @param pvCpuPage Pointer to the VMXON region.
4159	* @param HCPhysCpuPage Physical address of the VMXON region.
4160	*
4161	* @remarks This function should never be called when SUPR0EnableVTx() or
4162	* similar was used to enable VT-x on the host.
4163	*/
4164	VMMR0DECL(int) VMXR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4165	{
4166	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4167
4168	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4169	return hmR0VmxLeaveRootMode(pHostCpu);
4170	}
4171
4172
4173	/**
4174	* Does per-VM VT-x initialization.
4175	*
4176	* @returns VBox status code.
4177	* @param pVM The cross context VM structure.
4178	*/
4179	VMMR0DECL(int) VMXR0InitVM(PVMCC pVM)
4180	{
4181	AssertPtr(pVM);
4182	LogFlowFunc(("pVM=%p\n", pVM));
4183
4184	int rc = hmR0VmxStructsAlloc(pVM);
4185	if (RT_FAILURE(rc))
4186	{
4187	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4188	return rc;
4189	}
4190
4191	return VINF_SUCCESS;
4192	}
4193
4194
4195	/**
4196	* Does per-VM VT-x termination.
4197	*
4198	* @returns VBox status code.
4199	* @param pVM The cross context VM structure.
4200	*/
4201	VMMR0DECL(int) VMXR0TermVM(PVMCC pVM)
4202	{
4203	AssertPtr(pVM);
4204	LogFlowFunc(("pVM=%p\n", pVM));
4205
4206	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4207	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4208	{
4209	Assert(pVM->hm.s.vmx.pvScratch);
4210	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4211	}
4212	#endif
4213	hmR0VmxStructsFree(pVM);
4214	return VINF_SUCCESS;
4215	}
4216
4217
4218	/**
4219	* Sets up the VM for execution using hardware-assisted VMX.
4220	* This function is only called once per-VM during initialization.
4221	*
4222	* @returns VBox status code.
4223	* @param pVM The cross context VM structure.
4224	*/
4225	VMMR0DECL(int) VMXR0SetupVM(PVMCC pVM)
4226	{
4227	AssertPtr(pVM);
4228	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4229
4230	LogFlowFunc(("pVM=%p\n", pVM));
4231
4232	/*
4233	* At least verify if VMX is enabled, since we can't check if we're in VMX root mode or not
4234	* without causing a #GP.
4235	*/
4236	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4237	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4238	{ /* likely */ }
4239	else
4240	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4241
4242	/*
4243	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4244	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4245	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4246	*/
4247	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4248	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4249	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4250	{
4251	LogRelFunc(("Invalid real-on-v86 state.\n"));
4252	return VERR_INTERNAL_ERROR;
4253	}
4254
4255	/* Initialize these always, see hmR3InitFinalizeR0().*/
4256	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4257	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4258
4259	/* Setup the tagged-TLB flush handlers. */
4260	int rc = hmR0VmxSetupTaggedTlb(pVM);
4261	if (RT_FAILURE(rc))
4262	{
4263	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4264	return rc;
4265	}
4266
4267	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4268	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4269	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4270	{
4271	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4272	if (RT_SUCCESS(rc))
4273	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4274	else
4275	{
4276	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4277	return rc;
4278	}
4279	}
4280	#endif
4281
4282	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4283	{
4284	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
4285	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4286
4287	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4288	if (RT_SUCCESS(rc))
4289	{
4290	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4291	if (pVM->cpum.ro.GuestFeatures.fVmx)
4292	{
4293	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4294	if (RT_SUCCESS(rc))
4295	{ /* likely */ }
4296	else
4297	{
4298	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4299	return rc;
4300	}
4301	}
4302	#endif
4303	}
4304	else
4305	{
4306	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4307	return rc;
4308	}
4309	}
4310
4311	return VINF_SUCCESS;
4312	}
4313
4314
4315	/**
4316	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4317	* the VMCS.
4318	*/
4319	static void hmR0VmxExportHostControlRegs(void)
4320	{
4321	int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, ASMGetCR0()); AssertRC(rc);
4322	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, ASMGetCR3()); AssertRC(rc);
4323	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, ASMGetCR4()); AssertRC(rc);
4324	}
4325
4326
4327	/**
4328	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4329	* the host-state area in the VMCS.
4330	*
4331	* @returns VBox status code.
4332	* @param pVCpu The cross context virtual CPU structure.
4333	*/
4334	static int hmR0VmxExportHostSegmentRegs(PVMCPUCC pVCpu)
4335	{
4336	/**
4337	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4338	* requirements. See hmR0VmxExportHostSegmentRegs().
4339	*/
4340	#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_selValue) \
4341	if ((a_selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4342	{ \
4343	bool fValidSelector = true; \
4344	if ((a_selValue) & X86_SEL_LDT) \
4345	{ \
4346	uint32_t const uAttr = ASMGetSegAttr(a_selValue); \
4347	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4348	} \
4349	if (fValidSelector) \
4350	{ \
4351	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##a_Seg; \
4352	pVCpu->hm.s.vmx.RestoreHost.uHostSel##a_Seg = (a_selValue); \
4353	} \
4354	(a_selValue) = 0; \
4355	}
4356
4357	/*
4358	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4359	* will be messed up. We should -not- save the messed up state without restoring
4360	* the original host-state, see @bugref{7240}.
4361	*
4362	* This apparently can happen (most likely the FPU changes), deal with it rather than
4363	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4364	*/
4365	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4366	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4367	{
4368	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4369	pVCpu->idCpu));
4370	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4371	}
4372	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4373
4374	/*
4375	* Host segment registers.
4376	*/
4377	RTSEL uSelES = ASMGetES();
4378	RTSEL uSelCS = ASMGetCS();
4379	RTSEL uSelSS = ASMGetSS();
4380	RTSEL uSelDS = ASMGetDS();
4381	RTSEL uSelFS = ASMGetFS();
4382	RTSEL uSelGS = ASMGetGS();
4383	RTSEL uSelTR = ASMGetTR();
4384
4385	/*
4386	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4387	* gain VM-entry and restore them before we get preempted.
4388	*
4389	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4390	*/
4391	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4392	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4393	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4394	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4395
4396	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4397	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4398	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4399	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4400	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4401	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4402	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4403	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4404	Assert(uSelCS);
4405	Assert(uSelTR);
4406
4407	/* Write these host selector fields into the host-state area in the VMCS. */
4408	int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS); AssertRC(rc);
4409	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS); AssertRC(rc);
4410	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS); AssertRC(rc);
4411	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES); AssertRC(rc);
4412	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS); AssertRC(rc);
4413	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS); AssertRC(rc);
4414	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR); AssertRC(rc);
4415
4416	/*
4417	* Host GDTR and IDTR.
4418	*/
4419	RTGDTR Gdtr;
4420	RTIDTR Idtr;
4421	RT_ZERO(Gdtr);
4422	RT_ZERO(Idtr);
4423	ASMGetGDTR(&Gdtr);
4424	ASMGetIDTR(&Idtr);
4425	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRC(rc);
4426	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRC(rc);
4427
4428	/*
4429	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4430	* them to the maximum limit (0xffff) on every VM-exit.
4431	*/
4432	if (Gdtr.cbGdt != 0xffff)
4433	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4434
4435	/*
4436	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4437	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4438	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4439	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4440	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4441	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4442	* at 0xffff on hosts where we are sure it won't cause trouble.
4443	*/
4444	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4445	if (Idtr.cbIdt < 0x0fff)
4446	#else
4447	if (Idtr.cbIdt != 0xffff)
4448	#endif
4449	{
4450	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4451	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4452	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4453	}
4454
4455	/*
4456	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4457	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4458	* RPL should be too in most cases.
4459	*/
4460	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4461	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4462
4463	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4464	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4465
4466	/*
4467	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4468	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4469	* restoration if the host has something else. Task switching is not supported in 64-bit
4470	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4471	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4472	*
4473	* [1] See Intel spec. 3.5 "System Descriptor Types".
4474	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4475	*/
4476	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4477	Assert(pDesc->System.u4Type == 11);
4478	if ( pDesc->System.u16LimitLow != 0x67
4479	\|\| pDesc->System.u4LimitHigh)
4480	{
4481	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4482	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4483	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4484	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4485	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4486	}
4487
4488	/*
4489	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4490	*/
4491	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4492	{
4493	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4494	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4495	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4496	{
4497	/* The GDT is read-only but the writable GDT is available. */
4498	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4499	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4500	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4501	AssertRCReturn(rc, rc);
4502	}
4503	}
4504
4505	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase);
4506	AssertRC(rc);
4507
4508	/*
4509	* Host FS base and GS base.
4510	*/
4511	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4512	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4513	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRC(rc);
4514	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRC(rc);
4515
4516	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4517	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4518	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4519	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4520	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4521
4522	return VINF_SUCCESS;
4523	#undef VMXLOCAL_ADJUST_HOST_SEG
4524	}
4525
4526
4527	/**
4528	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4529	* host-state area of the VMCS.
4530	*
4531	* These MSRs will be automatically restored on the host after every successful
4532	* VM-exit.
4533	*
4534	* @param pVCpu The cross context virtual CPU structure.
4535	*
4536	* @remarks No-long-jump zone!!!
4537	*/
4538	static void hmR0VmxExportHostMsrs(PVMCPUCC pVCpu)
4539	{
4540	AssertPtr(pVCpu);
4541
4542	/*
4543	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4544	* rather than swapping them on every VM-entry.
4545	*/
4546	hmR0VmxLazySaveHostMsrs(pVCpu);
4547
4548	/*
4549	* Host Sysenter MSRs.
4550	*/
4551	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)); AssertRC(rc);
4552	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP)); AssertRC(rc);
4553	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP)); AssertRC(rc);
4554
4555	/*
4556	* Host EFER MSR.
4557	*
4558	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4559	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4560	*/
4561	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4562	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4563	{
4564	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4565	AssertRC(rc);
4566	}
4567
4568	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4569	* hmR0VmxExportGuestEntryExitCtls(). */
4570	}
4571
4572
4573	/**
4574	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4575	*
4576	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4577	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4578	*
4579	* @returns true if we need to load guest EFER, false otherwise.
4580	* @param pVCpu The cross context virtual CPU structure.
4581	* @param pVmxTransient The VMX-transient structure.
4582	*
4583	* @remarks Requires EFER, CR4.
4584	* @remarks No-long-jump zone!!!
4585	*/
4586	static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4587	{
4588	#ifdef HMVMX_ALWAYS_SWAP_EFER
4589	RT_NOREF2(pVCpu, pVmxTransient);
4590	return true;
4591	#else
4592	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4593	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4594	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4595	uint64_t const u64GuestEfer = pCtx->msrEFER;
4596
4597	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4598	/*
4599	* For nested-guests, we shall honor swapping the EFER MSR when requested by
4600	* the nested-guest.
4601	*/
4602	if ( pVmxTransient->fIsNestedGuest
4603	&& ( CPUMIsGuestVmxEntryCtlsSet(pVCpu, pCtx, VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4604	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_SAVE_EFER_MSR)
4605	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_LOAD_EFER_MSR)))
4606	return true;
4607	# else
4608	RT_NOREF(pVmxTransient);
4609	#endif
4610
4611	/*
4612	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4613	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4614	*/
4615	if ( CPUMIsGuestInLongModeEx(pCtx)
4616	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4617	return true;
4618
4619	/*
4620	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4621	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4622	*
4623	* See Intel spec. 4.5 "IA-32e Paging".
4624	* See Intel spec. 4.1.1 "Three Paging Modes".
4625	*
4626	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4627	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4628	*/
4629	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4630	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4631	if ( (pCtx->cr4 & X86_CR4_PAE)
4632	&& (pCtx->cr0 & X86_CR0_PG))
4633	{
4634	/*
4635	* If nested paging is not used, verify that the guest paging mode matches the
4636	* shadow paging mode which is/will be placed in the VMCS (which is what will
4637	* actually be used while executing the guest and not the CR4 shadow value).
4638	*/
4639	AssertMsg(pVM->hm.s.fNestedPaging \|\| ( pVCpu->hm.s.enmShadowMode == PGMMODE_PAE
4640	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_PAE_NX
4641	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64
4642	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64_NX),
4643	("enmShadowMode=%u\n", pVCpu->hm.s.enmShadowMode));
4644	if ((u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4645	{
4646	/* Verify that the host is NX capable. */
4647	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4648	return true;
4649	}
4650	}
4651
4652	return false;
4653	#endif
4654	}
4655
4656
4657	/**
4658	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4659	* VMCS.
4660	*
4661	* This is typically required when the guest changes paging mode.
4662	*
4663	* @returns VBox status code.
4664	* @param pVCpu The cross context virtual CPU structure.
4665	* @param pVmxTransient The VMX-transient structure.
4666	*
4667	* @remarks Requires EFER.
4668	* @remarks No-long-jump zone!!!
4669	*/
4670	static int hmR0VmxExportGuestEntryExitCtls(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4671	{
4672	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4673	{
4674	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4675	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4676
4677	/*
4678	* VM-entry controls.
4679	*/
4680	{
4681	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4682	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4683
4684	/*
4685	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4686	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4687	*
4688	* For nested-guests, this is a mandatory VM-entry control. It's also
4689	* required because we do not want to leak host bits to the nested-guest.
4690	*/
4691	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4692
4693	/*
4694	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4695	*
4696	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4697	* required to get the nested-guest working with hardware-assisted VMX execution.
4698	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
4699	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4700	* here rather than while merging the guest VMCS controls.
4701	*/
4702	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4703	{
4704	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
4705	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4706	}
4707	else
4708	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4709
4710	/*
4711	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4712	*
4713	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4714	* regardless of whether the nested-guest VMCS specifies it because we are free to
4715	* load whatever MSRs we require and we do not need to modify the guest visible copy
4716	* of the VM-entry MSR load area.
4717	*/
4718	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4719	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4720	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4721	else
4722	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4723
4724	/*
4725	* The following should -not- be set (since we're not in SMM mode):
4726	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4727	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4728	*/
4729
4730	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4731	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4732
4733	if ((fVal & fZap) == fVal)
4734	{ /* likely */ }
4735	else
4736	{
4737	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4738	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4739	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4740	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4741	}
4742
4743	/* Commit it to the VMCS. */
4744	if (pVmcsInfo->u32EntryCtls != fVal)
4745	{
4746	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4747	AssertRC(rc);
4748	pVmcsInfo->u32EntryCtls = fVal;
4749	}
4750	}
4751
4752	/*
4753	* VM-exit controls.
4754	*/
4755	{
4756	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4757	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4758
4759	/*
4760	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4761	* supported the 1-setting of this bit.
4762	*
4763	* For nested-guests, we set the "save debug controls" as the converse
4764	* "load debug controls" is mandatory for nested-guests anyway.
4765	*/
4766	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4767
4768	/*
4769	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4770	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4771	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4772	* hmR0VmxExportHostMsrs().
4773	*
4774	* For nested-guests, we always set this bit as we do not support 32-bit
4775	* hosts.
4776	*/
4777	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4778
4779	/*
4780	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4781	*
4782	* For nested-guests, we should use the "save IA32_EFER" control if we also
4783	* used the "load IA32_EFER" control while exporting VM-entry controls.
4784	*/
4785	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4786	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4787	{
4788	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4789	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4790	}
4791
4792	/*
4793	* Enable saving of the VMX-preemption timer value on VM-exit.
4794	* For nested-guests, currently not exposed/used.
4795	*/
4796	if ( pVM->hm.s.vmx.fUsePreemptTimer
4797	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4798	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4799
4800	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4801	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4802
4803	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4804	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4805	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4806
4807	if ((fVal & fZap) == fVal)
4808	{ /* likely */ }
4809	else
4810	{
4811	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4812	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4813	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4814	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4815	}
4816
4817	/* Commit it to the VMCS. */
4818	if (pVmcsInfo->u32ExitCtls != fVal)
4819	{
4820	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4821	AssertRC(rc);
4822	pVmcsInfo->u32ExitCtls = fVal;
4823	}
4824	}
4825
4826	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4827	}
4828	return VINF_SUCCESS;
4829	}
4830
4831
4832	/**
4833	* Sets the TPR threshold in the VMCS.
4834	*
4835	* @param pVmcsInfo The VMCS info. object.
4836	* @param u32TprThreshold The TPR threshold (task-priority class only).
4837	*/
4838	DECLINLINE(void) hmR0VmxApicSetTprThreshold(PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4839	{
4840	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4841	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4842	RT_NOREF(pVmcsInfo);
4843	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4844	AssertRC(rc);
4845	}
4846
4847
4848	/**
4849	* Exports the guest APIC TPR state into the VMCS.
4850	*
4851	* @returns VBox status code.
4852	* @param pVCpu The cross context virtual CPU structure.
4853	* @param pVmxTransient The VMX-transient structure.
4854	*
4855	* @remarks No-long-jump zone!!!
4856	*/
4857	static int hmR0VmxExportGuestApicTpr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4858	{
4859	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4860	{
4861	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4862
4863	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4864	if (!pVmxTransient->fIsNestedGuest)
4865	{
4866	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4867	&& APICIsEnabled(pVCpu))
4868	{
4869	/*
4870	* Setup TPR shadowing.
4871	*/
4872	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4873	{
4874	bool fPendingIntr = false;
4875	uint8_t u8Tpr = 0;
4876	uint8_t u8PendingIntr = 0;
4877	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4878	AssertRCReturn(rc, rc);
4879
4880	/*
4881	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4882	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4883	* priority of the pending interrupt so we can deliver the interrupt. If there
4884	* are no interrupts pending, set threshold to 0 to not cause any
4885	* TPR-below-threshold VM-exits.
4886	*/
4887	uint32_t u32TprThreshold = 0;
4888	if (fPendingIntr)
4889	{
4890	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4891	(which is the Task-Priority Class). */
4892	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4893	const uint8_t u8TprPriority = u8Tpr >> 4;
4894	if (u8PendingPriority <= u8TprPriority)
4895	u32TprThreshold = u8PendingPriority;
4896	}
4897
4898	hmR0VmxApicSetTprThreshold(pVmcsInfo, u32TprThreshold);
4899	}
4900	}
4901	}
4902	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4903	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4904	}
4905	return VINF_SUCCESS;
4906	}
4907
4908
4909	/**
4910	* Gets the guest interruptibility-state.
4911	*
4912	* @returns Guest's interruptibility-state.
4913	* @param pVCpu The cross context virtual CPU structure.
4914	* @param pVmxTransient The VMX-transient structure.
4915	*
4916	* @remarks No-long-jump zone!!!
4917	*/
4918	static uint32_t hmR0VmxGetGuestIntrState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4919	{
4920	/*
4921	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4922	*/
4923	uint32_t fIntrState = 0;
4924	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4925	{
4926	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
4927	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
4928
4929	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4930	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4931	{
4932	if (pCtx->eflags.Bits.u1IF)
4933	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4934	else
4935	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4936	}
4937	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4938	{
4939	/*
4940	* We can clear the inhibit force flag as even if we go back to the recompiler
4941	* without executing guest code in VT-x, the flag's condition to be cleared is
4942	* met and thus the cleared state is correct.
4943	*/
4944	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4945	}
4946	}
4947
4948	/*
4949	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4950	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4951	* setting this would block host-NMIs and IRET will not clear the blocking.
4952	*
4953	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4954	*
4955	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4956	*/
4957	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4958	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4959	&& CPUMIsGuestNmiBlocking(pVCpu))
4960	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4961
4962	return fIntrState;
4963	}
4964
4965
4966	/**
4967	* Exports the exception intercepts required for guest execution in the VMCS.
4968	*
4969	* @returns VBox status code.
4970	* @param pVCpu The cross context virtual CPU structure.
4971	* @param pVmxTransient The VMX-transient structure.
4972	*
4973	* @remarks No-long-jump zone!!!
4974	*/
4975	static int hmR0VmxExportGuestXcptIntercepts(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4976	{
4977	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4978	{
4979	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4980	if ( !pVmxTransient->fIsNestedGuest
4981	&& pVCpu->hm.s.fGIMTrapXcptUD)
4982	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4983	else
4984	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4985
4986	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4987	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4988	}
4989	return VINF_SUCCESS;
4990	}
4991
4992
4993	/**
4994	* Exports the guest's RIP into the guest-state area in the VMCS.
4995	*
4996	* @returns VBox status code.
4997	* @param pVCpu The cross context virtual CPU structure.
4998	*
4999	* @remarks No-long-jump zone!!!
5000	*/
5001	static int hmR0VmxExportGuestRip(PVMCPUCC pVCpu)
5002	{
5003	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5004	{
5005	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5006
5007	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5008	AssertRC(rc);
5009
5010	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5011	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5012	}
5013	return VINF_SUCCESS;
5014	}
5015
5016
5017	/**
5018	* Exports the guest's RSP into the guest-state area in the VMCS.
5019	*
5020	* @returns VBox status code.
5021	* @param pVCpu The cross context virtual CPU structure.
5022	*
5023	* @remarks No-long-jump zone!!!
5024	*/
5025	static int hmR0VmxExportGuestRsp(PVMCPUCC pVCpu)
5026	{
5027	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5028	{
5029	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5030
5031	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5032	AssertRC(rc);
5033
5034	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5035	Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
5036	}
5037	return VINF_SUCCESS;
5038	}
5039
5040
5041	/**
5042	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5043	*
5044	* @returns VBox status code.
5045	* @param pVCpu The cross context virtual CPU structure.
5046	* @param pVmxTransient The VMX-transient structure.
5047	*
5048	* @remarks No-long-jump zone!!!
5049	*/
5050	static int hmR0VmxExportGuestRflags(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5051	{
5052	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5053	{
5054	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5055
5056	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5057	Let us assert it as such and use 32-bit VMWRITE. */
5058	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5059	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5060	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5061	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5062
5063	/*
5064	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5065	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5066	* can run the real-mode guest code under Virtual 8086 mode.
5067	*/
5068	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5069	if (pVmcsInfo->RealMode.fRealOnV86Active)
5070	{
5071	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5072	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5073	Assert(!pVmxTransient->fIsNestedGuest);
5074	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5075	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5076	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5077	}
5078
5079	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5080	AssertRC(rc);
5081
5082	/*
5083	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5084	*
5085	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5086	* through the hypervisor debugger using EFLAGS.TF.
5087	*/
5088	if ( !pVmxTransient->fIsNestedGuest
5089	&& !pVCpu->hm.s.fSingleInstruction
5090	&& fEFlags.Bits.u1TF)
5091	{
5092	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5093	* premature trips to ring-3 esp since IEM does not yet handle it. */
5094	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5095	AssertRC(rc);
5096	}
5097	/* else: for nested-guest currently handling while merging controls. */
5098
5099	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5100	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5101	}
5102	return VINF_SUCCESS;
5103	}
5104
5105
5106	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5107	/**
5108	* Copies the nested-guest VMCS to the shadow VMCS.
5109	*
5110	* @returns VBox status code.
5111	* @param pVCpu The cross context virtual CPU structure.
5112	* @param pVmcsInfo The VMCS info. object.
5113	*
5114	* @remarks No-long-jump zone!!!
5115	*/
5116	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5117	{
5118	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5119	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5120
5121	/*
5122	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5123	* current VMCS, as we may try saving guest lazy MSRs.
5124	*
5125	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5126	* calling the import VMCS code which is currently performing the guest MSR reads
5127	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5128	* and the rest of the VMX leave session machinery.
5129	*/
5130	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5131
5132	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5133	if (RT_SUCCESS(rc))
5134	{
5135	/*
5136	* Copy all guest read/write VMCS fields.
5137	*
5138	* We don't check for VMWRITE failures here for performance reasons and
5139	* because they are not expected to fail, barring irrecoverable conditions
5140	* like hardware errors.
5141	*/
5142	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5143	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5144	{
5145	uint64_t u64Val;
5146	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5147	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5148	VMXWriteVmcs64(uVmcsField, u64Val);
5149	}
5150
5151	/*
5152	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5153	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5154	*/
5155	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5156	{
5157	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5158	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5159	{
5160	uint64_t u64Val;
5161	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5162	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5163	VMXWriteVmcs64(uVmcsField, u64Val);
5164	}
5165	}
5166
5167	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5168	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5169	}
5170
5171	ASMSetFlags(fEFlags);
5172	return rc;
5173	}
5174
5175
5176	/**
5177	* Copies the shadow VMCS to the nested-guest VMCS.
5178	*
5179	* @returns VBox status code.
5180	* @param pVCpu The cross context virtual CPU structure.
5181	* @param pVmcsInfo The VMCS info. object.
5182	*
5183	* @remarks Called with interrupts disabled.
5184	*/
5185	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5186	{
5187	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5188	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5189	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5190
5191	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5192	if (RT_SUCCESS(rc))
5193	{
5194	/*
5195	* Copy guest read/write fields from the shadow VMCS.
5196	* Guest read-only fields cannot be modified, so no need to copy them.
5197	*
5198	* We don't check for VMREAD failures here for performance reasons and
5199	* because they are not expected to fail, barring irrecoverable conditions
5200	* like hardware errors.
5201	*/
5202	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5203	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5204	{
5205	uint64_t u64Val;
5206	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5207	VMXReadVmcs64(uVmcsField, &u64Val);
5208	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5209	}
5210
5211	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5212	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5213	}
5214	return rc;
5215	}
5216
5217
5218	/**
5219	* Enables VMCS shadowing for the given VMCS info. object.
5220	*
5221	* @param pVmcsInfo The VMCS info. object.
5222	*
5223	* @remarks No-long-jump zone!!!
5224	*/
5225	static void hmR0VmxEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5226	{
5227	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5228	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5229	{
5230	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5231	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5232	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5233	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
5234	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5235	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5236	Log4Func(("Enabled\n"));
5237	}
5238	}
5239
5240
5241	/**
5242	* Disables VMCS shadowing for the given VMCS info. object.
5243	*
5244	* @param pVmcsInfo The VMCS info. object.
5245	*
5246	* @remarks No-long-jump zone!!!
5247	*/
5248	static void hmR0VmxDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5249	{
5250	/*
5251	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5252	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5253	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5254	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5255	*
5256	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5257	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5258	*/
5259	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5260	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5261	{
5262	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5263	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5264	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
5265	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5266	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5267	Log4Func(("Disabled\n"));
5268	}
5269	}
5270	#endif
5271
5272
5273	/**
5274	* Exports the guest hardware-virtualization state.
5275	*
5276	* @returns VBox status code.
5277	* @param pVCpu The cross context virtual CPU structure.
5278	* @param pVmxTransient The VMX-transient structure.
5279	*
5280	* @remarks No-long-jump zone!!!
5281	*/
5282	static int hmR0VmxExportGuestHwvirtState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5283	{
5284	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5285	{
5286	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5287	/*
5288	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5289	* VMCS shadowing.
5290	*/
5291	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5292	{
5293	/*
5294	* If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
5295	* copy the nested hypervisor's current VMCS into the shadow VMCS and enable
5296	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5297	*
5298	* We check for VMX root mode here in case the guest executes VMXOFF without
5299	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5300	* not clear the current VMCS pointer.
5301	*/
5302	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5303	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5304	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5305	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5306	{
5307	/* Paranoia. */
5308	Assert(!pVmxTransient->fIsNestedGuest);
5309
5310	/*
5311	* For performance reasons, also check if the nested hypervisor's current VMCS
5312	* was newly loaded or modified before copying it to the shadow VMCS.
5313	*/
5314	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5315	{
5316	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5317	AssertRCReturn(rc, rc);
5318	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5319	}
5320	hmR0VmxEnableVmcsShadowing(pVmcsInfo);
5321	}
5322	else
5323	hmR0VmxDisableVmcsShadowing(pVmcsInfo);
5324	}
5325	#else
5326	NOREF(pVmxTransient);
5327	#endif
5328	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5329	}
5330	return VINF_SUCCESS;
5331	}
5332
5333
5334	/**
5335	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5336	*
5337	* The guest FPU state is always pre-loaded hence we don't need to bother about
5338	* sharing FPU related CR0 bits between the guest and host.
5339	*
5340	* @returns VBox status code.
5341	* @param pVCpu The cross context virtual CPU structure.
5342	* @param pVmxTransient The VMX-transient structure.
5343	*
5344	* @remarks No-long-jump zone!!!
5345	*/
5346	static int hmR0VmxExportGuestCR0(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5347	{
5348	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5349	{
5350	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5351	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5352
5353	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0;
5354	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5355	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5356	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5357	else
5358	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5359
5360	if (!pVmxTransient->fIsNestedGuest)
5361	{
5362	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5363	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5364	uint64_t const u64ShadowCr0 = u64GuestCr0;
5365	Assert(!RT_HI_U32(u64GuestCr0));
5366
5367	/*
5368	* Setup VT-x's view of the guest CR0.
5369	*/
5370	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5371	if (pVM->hm.s.fNestedPaging)
5372	{
5373	if (CPUMIsGuestPagingEnabled(pVCpu))
5374	{
5375	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5376	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5377	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5378	}
5379	else
5380	{
5381	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5382	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5383	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5384	}
5385
5386	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5387	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5388	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5389	}
5390	else
5391	{
5392	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5393	u64GuestCr0 \|= X86_CR0_WP;
5394	}
5395
5396	/*
5397	* Guest FPU bits.
5398	*
5399	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5400	* using CR0.TS.
5401	*
5402	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5403	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5404	*/
5405	u64GuestCr0 \|= X86_CR0_NE;
5406
5407	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5408	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5409
5410	/*
5411	* Update exception intercepts.
5412	*/
5413	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5414	if (pVmcsInfo->RealMode.fRealOnV86Active)
5415	{
5416	Assert(PDMVmmDevHeapIsEnabled(pVM));
5417	Assert(pVM->hm.s.vmx.pRealModeTSS);
5418	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5419	}
5420	else
5421	{
5422	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5423	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5424	if (fInterceptMF)
5425	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5426	}
5427
5428	/* Additional intercepts for debugging, define these yourself explicitly. */
5429	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5430	uXcptBitmap \|= 0
5431	\| RT_BIT(X86_XCPT_BP)
5432	\| RT_BIT(X86_XCPT_DE)
5433	\| RT_BIT(X86_XCPT_NM)
5434	\| RT_BIT(X86_XCPT_TS)
5435	\| RT_BIT(X86_XCPT_UD)
5436	\| RT_BIT(X86_XCPT_NP)
5437	\| RT_BIT(X86_XCPT_SS)
5438	\| RT_BIT(X86_XCPT_GP)
5439	\| RT_BIT(X86_XCPT_PF)
5440	\| RT_BIT(X86_XCPT_MF)
5441	;
5442	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5443	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5444	#endif
5445	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5446	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5447	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5448
5449	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5450	u64GuestCr0 \|= fSetCr0;
5451	u64GuestCr0 &= fZapCr0;
5452	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5453
5454	/* Commit the CR0 and related fields to the guest VMCS. */
5455	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5456	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5457	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5458	{
5459	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5460	AssertRC(rc);
5461	}
5462	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5463	{
5464	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5465	AssertRC(rc);
5466	}
5467
5468	/* Update our caches. */
5469	pVmcsInfo->u32ProcCtls = uProcCtls;
5470	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5471
5472	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5473	}
5474	else
5475	{
5476	/*
5477	* With nested-guests, we may have extended the guest/host mask here since we
5478	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5479	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
5480	* originally supplied. We must copy those bits from the nested-guest CR0 into
5481	* the nested-guest CR0 read-shadow.
5482	*/
5483	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5484	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5485	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5486	Assert(!RT_HI_U32(u64GuestCr0));
5487	Assert(u64GuestCr0 & X86_CR0_NE);
5488
5489	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5490	u64GuestCr0 \|= fSetCr0;
5491	u64GuestCr0 &= fZapCr0;
5492	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5493
5494	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5495	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5496	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5497
5498	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5499	}
5500
5501	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5502	}
5503
5504	return VINF_SUCCESS;
5505	}
5506
5507
5508	/**
5509	* Exports the guest control registers (CR3, CR4) into the guest-state area
5510	* in the VMCS.
5511	*
5512	* @returns VBox strict status code.
5513	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5514	* without unrestricted guest access and the VMMDev is not presently
5515	* mapped (e.g. EFI32).
5516	*
5517	* @param pVCpu The cross context virtual CPU structure.
5518	* @param pVmxTransient The VMX-transient structure.
5519	*
5520	* @remarks No-long-jump zone!!!
5521	*/
5522	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5523	{
5524	int rc = VINF_SUCCESS;
5525	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5526
5527	/*
5528	* Guest CR2.
5529	* It's always loaded in the assembler code. Nothing to do here.
5530	*/
5531
5532	/*
5533	* Guest CR3.
5534	*/
5535	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5536	{
5537	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5538
5539	if (pVM->hm.s.fNestedPaging)
5540	{
5541	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5542	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5543
5544	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5545	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5546	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5547	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5548
5549	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5550	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5551	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5552
5553	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5554	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5555	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5556	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5557	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5558	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5559	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5560
5561	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5562	AssertRC(rc);
5563
5564	uint64_t u64GuestCr3;
5565	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5566	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5567	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5568	{
5569	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5570	if (CPUMIsGuestInPAEModeEx(pCtx))
5571	{
5572	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5573	AssertRC(rc);
5574	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
5575	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
5576	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
5577	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
5578	}
5579
5580	/*
5581	* The guest's view of its CR3 is unblemished with nested paging when the
5582	* guest is using paging or we have unrestricted guest execution to handle
5583	* the guest when it's not using paging.
5584	*/
5585	u64GuestCr3 = pCtx->cr3;
5586	}
5587	else
5588	{
5589	/*
5590	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5591	* thinks it accesses physical memory directly, we use our identity-mapped
5592	* page table to map guest-linear to guest-physical addresses. EPT takes care
5593	* of translating it to host-physical addresses.
5594	*/
5595	RTGCPHYS GCPhys;
5596	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5597
5598	/* We obtain it here every time as the guest could have relocated this PCI region. */
5599	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5600	if (RT_SUCCESS(rc))
5601	{ /* likely */ }
5602	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5603	{
5604	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5605	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5606	}
5607	else
5608	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5609
5610	u64GuestCr3 = GCPhys;
5611	}
5612
5613	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
5614	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, u64GuestCr3);
5615	AssertRC(rc);
5616	}
5617	else
5618	{
5619	Assert(!pVmxTransient->fIsNestedGuest);
5620	/* Non-nested paging case, just use the hypervisor's CR3. */
5621	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5622
5623	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
5624	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5625	AssertRC(rc);
5626	}
5627
5628	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5629	}
5630
5631	/*
5632	* Guest CR4.
5633	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5634	*/
5635	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5636	{
5637	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5638	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5639
5640	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0;
5641	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5642
5643	/*
5644	* With nested-guests, we may have extended the guest/host mask here (since we
5645	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5646	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5647	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
5648	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5649	*/
5650	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5651	uint64_t u64GuestCr4 = pCtx->cr4;
5652	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5653	? pCtx->cr4
5654	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5655	Assert(!RT_HI_U32(u64GuestCr4));
5656
5657	/*
5658	* Setup VT-x's view of the guest CR4.
5659	*
5660	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5661	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5662	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5663	*
5664	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5665	*/
5666	if (pVmcsInfo->RealMode.fRealOnV86Active)
5667	{
5668	Assert(pVM->hm.s.vmx.pRealModeTSS);
5669	Assert(PDMVmmDevHeapIsEnabled(pVM));
5670	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5671	}
5672
5673	if (pVM->hm.s.fNestedPaging)
5674	{
5675	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5676	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5677	{
5678	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5679	u64GuestCr4 \|= X86_CR4_PSE;
5680	/* Our identity mapping is a 32-bit page directory. */
5681	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5682	}
5683	/* else use guest CR4.*/
5684	}
5685	else
5686	{
5687	Assert(!pVmxTransient->fIsNestedGuest);
5688
5689	/*
5690	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5691	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5692	*/
5693	switch (pVCpu->hm.s.enmShadowMode)
5694	{
5695	case PGMMODE_REAL: /* Real-mode. */
5696	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5697	case PGMMODE_32_BIT: /* 32-bit paging. */
5698	{
5699	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5700	break;
5701	}
5702
5703	case PGMMODE_PAE: /* PAE paging. */
5704	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5705	{
5706	u64GuestCr4 \|= X86_CR4_PAE;
5707	break;
5708	}
5709
5710	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5711	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5712	{
5713	#ifdef VBOX_WITH_64_BITS_GUESTS
5714	/* For our assumption in hmR0VmxShouldSwapEferMsr. */
5715	Assert(u64GuestCr4 & X86_CR4_PAE);
5716	break;
5717	#endif
5718	}
5719	default:
5720	AssertFailed();
5721	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5722	}
5723	}
5724
5725	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
5726	u64GuestCr4 \|= fSetCr4;
5727	u64GuestCr4 &= fZapCr4;
5728
5729	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5730	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
5731	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
5732
5733	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5734	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5735
5736	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5737
5738	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5739	}
5740	return rc;
5741	}
5742
5743
5744	/**
5745	* Exports the guest debug registers into the guest-state area in the VMCS.
5746	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5747	*
5748	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5749	*
5750	* @returns VBox status code.
5751	* @param pVCpu The cross context virtual CPU structure.
5752	* @param pVmxTransient The VMX-transient structure.
5753	*
5754	* @remarks No-long-jump zone!!!
5755	*/
5756	static int hmR0VmxExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5757	{
5758	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5759
5760	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5761	* stepping. */
5762	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5763	if (pVmxTransient->fIsNestedGuest)
5764	{
5765	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5766	AssertRC(rc);
5767
5768	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5769	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5770	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5771	AssertRC(rc);
5772	return VINF_SUCCESS;
5773	}
5774
5775	#ifdef VBOX_STRICT
5776	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5777	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5778	{
5779	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5780	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5781	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5782	}
5783	#endif
5784
5785	bool fSteppingDB = false;
5786	bool fInterceptMovDRx = false;
5787	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5788	if (pVCpu->hm.s.fSingleInstruction)
5789	{
5790	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5791	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5792	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5793	{
5794	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5795	Assert(fSteppingDB == false);
5796	}
5797	else
5798	{
5799	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5800	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5801	pVCpu->hm.s.fClearTrapFlag = true;
5802	fSteppingDB = true;
5803	}
5804	}
5805
5806	uint64_t u64GuestDr7;
5807	if ( fSteppingDB
5808	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5809	{
5810	/*
5811	* Use the combined guest and host DRx values found in the hypervisor register set
5812	* because the hypervisor debugger has breakpoints active or someone is single stepping
5813	* on the host side without a monitor trap flag.
5814	*
5815	* Note! DBGF expects a clean DR6 state before executing guest code.
5816	*/
5817	if (!CPUMIsHyperDebugStateActive(pVCpu))
5818	{
5819	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5820	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5821	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5822	}
5823
5824	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5825	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
5826	pVCpu->hm.s.fUsingHyperDR7 = true;
5827	fInterceptMovDRx = true;
5828	}
5829	else
5830	{
5831	/*
5832	* If the guest has enabled debug registers, we need to load them prior to
5833	* executing guest code so they'll trigger at the right time.
5834	*/
5835	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5836	{
5837	if (!CPUMIsGuestDebugStateActive(pVCpu))
5838	{
5839	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5840	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5841	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5842	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5843	}
5844	Assert(!fInterceptMovDRx);
5845	}
5846	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5847	{
5848	/*
5849	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5850	* must intercept #DB in order to maintain a correct DR6 guest value, and
5851	* because we need to intercept it to prevent nested #DBs from hanging the
5852	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5853	*/
5854	fInterceptMovDRx = true;
5855	}
5856
5857	/* Update DR7 with the actual guest value. */
5858	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5859	pVCpu->hm.s.fUsingHyperDR7 = false;
5860	}
5861
5862	if (fInterceptMovDRx)
5863	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5864	else
5865	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5866
5867	/*
5868	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5869	* monitor-trap flag and update our cache.
5870	*/
5871	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5872	{
5873	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5874	AssertRC(rc);
5875	pVmcsInfo->u32ProcCtls = uProcCtls;
5876	}
5877
5878	/*
5879	* Update guest DR7.
5880	*/
5881	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
5882	AssertRC(rc);
5883
5884	/*
5885	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5886	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5887	*
5888	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5889	*/
5890	if (fSteppingDB)
5891	{
5892	Assert(pVCpu->hm.s.fSingleInstruction);
5893	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5894
5895	uint32_t fIntrState = 0;
5896	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5897	AssertRC(rc);
5898
5899	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5900	{
5901	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5902	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5903	AssertRC(rc);
5904	}
5905	}
5906
5907	return VINF_SUCCESS;
5908	}
5909
5910
5911	#ifdef VBOX_STRICT
5912	/**
5913	* Strict function to validate segment registers.
5914	*
5915	* @param pVCpu The cross context virtual CPU structure.
5916	* @param pVmcsInfo The VMCS info. object.
5917	*
5918	* @remarks Will import guest CR0 on strict builds during validation of
5919	* segments.
5920	*/
5921	static void hmR0VmxValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5922	{
5923	/*
5924	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5925	*
5926	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5927	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5928	* unusable bit and doesn't change the guest-context value.
5929	*/
5930	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5931	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5932	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5933	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5934	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5935	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5936	{
5937	/* Protected mode checks */
5938	/* CS */
5939	Assert(pCtx->cs.Attr.n.u1Present);
5940	Assert(!(pCtx->cs.Attr.u & 0xf00));
5941	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5942	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5943	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5944	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5945	\|\| (pCtx->cs.Attr.n.u1Granularity));
5946	/* CS cannot be loaded with NULL in protected mode. */
5947	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5948	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5949	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5950	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5951	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5952	else
5953	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5954	/* SS */
5955	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5956	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5957	if ( !(pCtx->cr0 & X86_CR0_PE)
5958	\|\| pCtx->cs.Attr.n.u4Type == 3)
5959	{
5960	Assert(!pCtx->ss.Attr.n.u2Dpl);
5961	}
5962	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5963	{
5964	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5965	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5966	Assert(pCtx->ss.Attr.n.u1Present);
5967	Assert(!(pCtx->ss.Attr.u & 0xf00));
5968	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5969	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5970	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5971	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5972	\|\| (pCtx->ss.Attr.n.u1Granularity));
5973	}
5974	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5975	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5976	{
5977	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5978	Assert(pCtx->ds.Attr.n.u1Present);
5979	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5980	Assert(!(pCtx->ds.Attr.u & 0xf00));
5981	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5982	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5983	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5984	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5985	\|\| (pCtx->ds.Attr.n.u1Granularity));
5986	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5987	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5988	}
5989	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5990	{
5991	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5992	Assert(pCtx->es.Attr.n.u1Present);
5993	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5994	Assert(!(pCtx->es.Attr.u & 0xf00));
5995	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5996	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5997	\|\| !(pCtx->es.Attr.n.u1Granularity));
5998	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5999	\|\| (pCtx->es.Attr.n.u1Granularity));
6000	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6001	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6002	}
6003	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6004	{
6005	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6006	Assert(pCtx->fs.Attr.n.u1Present);
6007	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6008	Assert(!(pCtx->fs.Attr.u & 0xf00));
6009	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6010	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6011	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6012	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6013	\|\| (pCtx->fs.Attr.n.u1Granularity));
6014	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6015	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6016	}
6017	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6018	{
6019	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6020	Assert(pCtx->gs.Attr.n.u1Present);
6021	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6022	Assert(!(pCtx->gs.Attr.u & 0xf00));
6023	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6024	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6025	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6026	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6027	\|\| (pCtx->gs.Attr.n.u1Granularity));
6028	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6029	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6030	}
6031	/* 64-bit capable CPUs. */
6032	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6033	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6034	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6035	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6036	}
6037	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6038	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6039	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6040	{
6041	/* Real and v86 mode checks. */
6042	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6043	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6044	if (pVmcsInfo->RealMode.fRealOnV86Active)
6045	{
6046	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6047	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6048	}
6049	else
6050	{
6051	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6052	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6053	}
6054
6055	/* CS */
6056	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6057	Assert(pCtx->cs.u32Limit == 0xffff);
6058	Assert(u32CSAttr == 0xf3);
6059	/* SS */
6060	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6061	Assert(pCtx->ss.u32Limit == 0xffff);
6062	Assert(u32SSAttr == 0xf3);
6063	/* DS */
6064	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6065	Assert(pCtx->ds.u32Limit == 0xffff);
6066	Assert(u32DSAttr == 0xf3);
6067	/* ES */
6068	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6069	Assert(pCtx->es.u32Limit == 0xffff);
6070	Assert(u32ESAttr == 0xf3);
6071	/* FS */
6072	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6073	Assert(pCtx->fs.u32Limit == 0xffff);
6074	Assert(u32FSAttr == 0xf3);
6075	/* GS */
6076	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6077	Assert(pCtx->gs.u32Limit == 0xffff);
6078	Assert(u32GSAttr == 0xf3);
6079	/* 64-bit capable CPUs. */
6080	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6081	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6082	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6083	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6084	}
6085	}
6086	#endif /* VBOX_STRICT */
6087
6088
6089	/**
6090	* Exports a guest segment register into the guest-state area in the VMCS.
6091	*
6092	* @returns VBox status code.
6093	* @param pVCpu The cross context virtual CPU structure.
6094	* @param pVmcsInfo The VMCS info. object.
6095	* @param iSegReg The segment register number (X86_SREG_XXX).
6096	* @param pSelReg Pointer to the segment selector.
6097	*
6098	* @remarks No-long-jump zone!!!
6099	*/
6100	static int hmR0VmxExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6101	{
6102	Assert(iSegReg < X86_SREG_COUNT);
6103	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6104	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6105	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6106	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6107
6108	uint32_t u32Access = pSelReg->Attr.u;
6109	if (pVmcsInfo->RealMode.fRealOnV86Active)
6110	{
6111	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6112	u32Access = 0xf3;
6113	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6114	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6115	RT_NOREF_PV(pVCpu);
6116	}
6117	else
6118	{
6119	/*
6120	* The way to differentiate between whether this is really a null selector or was just
6121	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6122	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6123	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6124	* NULL selectors loaded in protected-mode have their attribute as 0.
6125	*/
6126	if (!u32Access)
6127	u32Access = X86DESCATTR_UNUSABLE;
6128	}
6129
6130	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6131	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6132	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6133
6134	/*
6135	* Commit it to the VMCS.
6136	*/
6137	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); AssertRC(rc);
6138	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); AssertRC(rc);
6139	rc = VMXWriteVmcsNw(idxBase, pSelReg->u64Base); AssertRC(rc);
6140	rc = VMXWriteVmcs32(idxAttr, u32Access); AssertRC(rc);
6141	return VINF_SUCCESS;
6142	}
6143
6144
6145	/**
6146	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6147	* area in the VMCS.
6148	*
6149	* @returns VBox status code.
6150	* @param pVCpu The cross context virtual CPU structure.
6151	* @param pVmxTransient The VMX-transient structure.
6152	*
6153	* @remarks Will import guest CR0 on strict builds during validation of
6154	* segments.
6155	* @remarks No-long-jump zone!!!
6156	*/
6157	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6158	{
6159	int rc = VERR_INTERNAL_ERROR_5;
6160	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6161	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6162	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6163
6164	/*
6165	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6166	*/
6167	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6168	{
6169	#ifdef VBOX_WITH_REM
6170	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6171	{
6172	Assert(!pVmxTransient->fIsNestedGuest);
6173	Assert(pVM->hm.s.vmx.pRealModeTSS);
6174	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6175	if ( pVmcsInfo->fWasInRealMode
6176	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6177	{
6178	/*
6179	* Notify the recompiler must flush its code-cache as the guest -may-
6180	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6181	*/
6182	REMFlushTBs(pVM);
6183	Log4Func(("Switch to protected mode detected!\n"));
6184	pVmcsInfo->fWasInRealMode = false;
6185	}
6186	}
6187	#endif
6188	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6189	{
6190	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6191	if (pVmcsInfo->RealMode.fRealOnV86Active)
6192	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6193	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6194	AssertRC(rc);
6195	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6196	}
6197
6198	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6199	{
6200	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6201	if (pVmcsInfo->RealMode.fRealOnV86Active)
6202	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6203	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6204	AssertRC(rc);
6205	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6206	}
6207
6208	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6209	{
6210	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6211	if (pVmcsInfo->RealMode.fRealOnV86Active)
6212	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6213	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6214	AssertRC(rc);
6215	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6216	}
6217
6218	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6219	{
6220	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6221	if (pVmcsInfo->RealMode.fRealOnV86Active)
6222	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6223	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6224	AssertRC(rc);
6225	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6226	}
6227
6228	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6229	{
6230	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6231	if (pVmcsInfo->RealMode.fRealOnV86Active)
6232	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6233	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6234	AssertRC(rc);
6235	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6236	}
6237
6238	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6239	{
6240	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6241	if (pVmcsInfo->RealMode.fRealOnV86Active)
6242	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6243	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6244	AssertRC(rc);
6245	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6246	}
6247
6248	#ifdef VBOX_STRICT
6249	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6250	#endif
6251	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6252	pCtx->cs.Attr.u));
6253	}
6254
6255	/*
6256	* Guest TR.
6257	*/
6258	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6259	{
6260	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6261
6262	/*
6263	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6264	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6265	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6266	*/
6267	uint16_t u16Sel;
6268	uint32_t u32Limit;
6269	uint64_t u64Base;
6270	uint32_t u32AccessRights;
6271	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6272	{
6273	u16Sel = pCtx->tr.Sel;
6274	u32Limit = pCtx->tr.u32Limit;
6275	u64Base = pCtx->tr.u64Base;
6276	u32AccessRights = pCtx->tr.Attr.u;
6277	}
6278	else
6279	{
6280	Assert(!pVmxTransient->fIsNestedGuest);
6281	Assert(pVM->hm.s.vmx.pRealModeTSS);
6282	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6283
6284	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6285	RTGCPHYS GCPhys;
6286	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6287	AssertRCReturn(rc, rc);
6288
6289	X86DESCATTR DescAttr;
6290	DescAttr.u = 0;
6291	DescAttr.n.u1Present = 1;
6292	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6293
6294	u16Sel = 0;
6295	u32Limit = HM_VTX_TSS_SIZE;
6296	u64Base = GCPhys;
6297	u32AccessRights = DescAttr.u;
6298	}
6299
6300	/* Validate. */
6301	Assert(!(u16Sel & RT_BIT(2)));
6302	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6303	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6304	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6305	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6306	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6307	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6308	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6309	Assert( (u32Limit & 0xfff) == 0xfff
6310	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6311	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6312	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6313
6314	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
6315	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
6316	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
6317	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
6318
6319	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6320	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6321	}
6322
6323	/*
6324	* Guest GDTR.
6325	*/
6326	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6327	{
6328	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6329
6330	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
6331	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
6332
6333	/* Validate. */
6334	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6335
6336	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6337	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6338	}
6339
6340	/*
6341	* Guest LDTR.
6342	*/
6343	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6344	{
6345	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6346
6347	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6348	uint32_t u32Access;
6349	if ( !pVmxTransient->fIsNestedGuest
6350	&& !pCtx->ldtr.Attr.u)
6351	u32Access = X86DESCATTR_UNUSABLE;
6352	else
6353	u32Access = pCtx->ldtr.Attr.u;
6354
6355	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
6356	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
6357	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
6358	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
6359
6360	/* Validate. */
6361	if (!(u32Access & X86DESCATTR_UNUSABLE))
6362	{
6363	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6364	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6365	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6366	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6367	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6368	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6369	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6370	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6371	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6372	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6373	}
6374
6375	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6376	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6377	}
6378
6379	/*
6380	* Guest IDTR.
6381	*/
6382	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6383	{
6384	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6385
6386	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
6387	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
6388
6389	/* Validate. */
6390	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6391
6392	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6393	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6394	}
6395
6396	return VINF_SUCCESS;
6397	}
6398
6399
6400	/**
6401	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6402	* areas.
6403	*
6404	* These MSRs will automatically be loaded to the host CPU on every successful
6405	* VM-entry and stored from the host CPU on every successful VM-exit.
6406	*
6407	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6408	* actual host MSR values are not- updated here for performance reasons. See
6409	* hmR0VmxExportHostMsrs().
6410	*
6411	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6412	*
6413	* @returns VBox status code.
6414	* @param pVCpu The cross context virtual CPU structure.
6415	* @param pVmxTransient The VMX-transient structure.
6416	*
6417	* @remarks No-long-jump zone!!!
6418	*/
6419	static int hmR0VmxExportGuestMsrs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6420	{
6421	AssertPtr(pVCpu);
6422	AssertPtr(pVmxTransient);
6423
6424	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6425	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6426
6427	/*
6428	* MSRs that we use the auto-load/store MSR area in the VMCS.
6429	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
6430	* nothing to do here. The host MSR values are updated when it's safe in
6431	* hmR0VmxLazySaveHostMsrs().
6432	*
6433	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6434	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6435	* emulation. The merged MSR permission bitmap will ensure that we get VM-exits
6436	* for any MSR that are not part of the lazy MSRs so we do not need to place
6437	* those MSRs into the auto-load/store MSR area. Nothing to do here.
6438	*/
6439	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6440	{
6441	/* No auto-load/store MSRs currently. */
6442	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6443	}
6444
6445	/*
6446	* Guest Sysenter MSRs.
6447	*/
6448	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6449	{
6450	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6451
6452	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6453	{
6454	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6455	AssertRC(rc);
6456	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6457	}
6458
6459	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6460	{
6461	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6462	AssertRC(rc);
6463	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6464	}
6465
6466	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6467	{
6468	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6469	AssertRC(rc);
6470	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6471	}
6472	}
6473
6474	/*
6475	* Guest/host EFER MSR.
6476	*/
6477	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6478	{
6479	/* Whether we are using the VMCS to swap the EFER MSR must have been
6480	determined earlier while exporting VM-entry/VM-exit controls. */
6481	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6482	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6483
6484	if (hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
6485	{
6486	/*
6487	* EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
6488	* This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
6489	* VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
6490	* and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
6491	* must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
6492	* the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
6493	* during VM-entry.
6494	*/
6495	uint64_t uGuestEferMsr = pCtx->msrEFER;
6496	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6497	{
6498	if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
6499	uGuestEferMsr &= ~MSR_K6_EFER_LME;
6500	else
6501	Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
6502	}
6503
6504	/*
6505	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6506	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6507	*/
6508	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6509	{
6510	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
6511	AssertRC(rc);
6512	}
6513	else
6514	{
6515	/*
6516	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6517	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6518	*/
6519	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
6520	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6521	AssertRCReturn(rc, rc);
6522	}
6523
6524	Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
6525	}
6526	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6527	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6528
6529	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6530	}
6531
6532	/*
6533	* Other MSRs.
6534	* Speculation Control (R/W).
6535	*/
6536	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6537	{
6538	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6539	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6540	{
6541	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6542	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6543	AssertRCReturn(rc, rc);
6544	}
6545	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6546	}
6547
6548	return VINF_SUCCESS;
6549	}
6550
6551
6552	/**
6553	* Selects up the appropriate function to run guest code.
6554	*
6555	* @returns VBox status code.
6556	* @param pVCpu The cross context virtual CPU structure.
6557	* @param pVmxTransient The VMX-transient structure.
6558	*
6559	* @remarks No-long-jump zone!!!
6560	*/
6561	static int hmR0VmxSelectVMRunHandler(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6562	{
6563	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6564	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6565
6566	if (CPUMIsGuestInLongModeEx(pCtx))
6567	{
6568	#ifndef VBOX_WITH_64_BITS_GUESTS
6569	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6570	#else
6571	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6572	/* Guest is in long mode, use the 64-bit handler (host is 64-bit). */
6573	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6574	#endif
6575	}
6576	else
6577	{
6578	/* Guest is not in long mode, use the 32-bit handler. */
6579	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6580	}
6581	Assert(pVmcsInfo->pfnStartVM);
6582	return VINF_SUCCESS;
6583	}
6584
6585
6586	/**
6587	* Wrapper for running the guest code in VT-x.
6588	*
6589	* @returns VBox status code, no informational status codes.
6590	* @param pVCpu The cross context virtual CPU structure.
6591	* @param pVmxTransient The VMX-transient structure.
6592	*
6593	* @remarks No-long-jump zone!!!
6594	*/
6595	DECLINLINE(int) hmR0VmxRunGuest(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6596	{
6597	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6598	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6599	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6600
6601	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6602
6603	/*
6604	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6605	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6606	* callee-saved and thus the need for this XMM wrapper.
6607	*
6608	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6609	*/
6610	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6611	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6612	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6613	#ifdef VBOX_WITH_KERNEL_USING_XMM
6614	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6615	#else
6616	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu);
6617	#endif
6618	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6619	return rc;
6620	}
6621
6622
6623	/**
6624	* Reports world-switch error and dumps some useful debug info.
6625	*
6626	* @param pVCpu The cross context virtual CPU structure.
6627	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6628	* @param pVmxTransient The VMX-transient structure (only
6629	* exitReason updated).
6630	*/
6631	static void hmR0VmxReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6632	{
6633	Assert(pVCpu);
6634	Assert(pVmxTransient);
6635	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6636
6637	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6638	switch (rcVMRun)
6639	{
6640	case VERR_VMX_INVALID_VMXON_PTR:
6641	AssertFailed();
6642	break;
6643	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6644	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6645	{
6646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6647	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6648	AssertRC(rc);
6649	hmR0VmxReadExitQualVmcs(pVmxTransient);
6650
6651	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6652	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6653	Cannot do it here as we may have been long preempted. */
6654
6655	#ifdef VBOX_STRICT
6656	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6657	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6658	pVmxTransient->uExitReason));
6659	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6660	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6661	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6662	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6663	else
6664	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6665	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6666	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6667
6668	static struct
6669	{
6670	/** Name of the field to log. */
6671	const char *pszName;
6672	/** The VMCS field. */
6673	uint32_t uVmcsField;
6674	/** Whether host support of this field needs to be checked. */
6675	bool fCheckSupport;
6676	} const s_aVmcsFields[] =
6677	{
6678	{ "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false },
6679	{ "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false },
6680	{ "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true },
6681	{ "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false },
6682	{ "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false },
6683	{ "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false },
6684	{ "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false },
6685	{ "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false },
6686	{ "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false },
6687	{ "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false },
6688	{ "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false },
6689	{ "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false },
6690	{ "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false },
6691	{ "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false },
6692	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false },
6693	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false },
6694	{ "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false },
6695	{ "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false },
6696	{ "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false },
6697	{ "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false },
6698	{ "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true },
6699	{ "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false },
6700	{ "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false },
6701	{ "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false },
6702	{ "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
6703	{ "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false },
6704	{ "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false },
6705	{ "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false },
6706	/* The order of selector fields below are fixed! */
6707	{ "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false },
6708	{ "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false },
6709	{ "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false },
6710	{ "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false },
6711	{ "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false },
6712	{ "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false },
6713	{ "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false },
6714	/* End of ordered selector fields. */
6715	{ "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false },
6716	{ "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false },
6717	{ "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false },
6718	{ "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false },
6719	{ "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false },
6720	{ "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false },
6721	{ "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false },
6722	{ "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false }
6723	};
6724
6725	RTGDTR HostGdtr;
6726	ASMGetGDTR(&HostGdtr);
6727
6728	uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
6729	for (uint32_t i = 0; i < cVmcsFields; i++)
6730	{
6731	uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
6732
6733	bool fSupported;
6734	if (!s_aVmcsFields[i].fCheckSupport)
6735	fSupported = true;
6736	else
6737	{
6738	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6739	switch (uVmcsField)
6740	{
6741	case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hm.s.fNestedPaging; break;
6742	case VMX_VMCS16_VPID: fSupported = pVM->hm.s.vmx.fVpid; break;
6743	case VMX_VMCS32_CTRL_PROC_EXEC2:
6744	fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6745	break;
6746	default:
6747	AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
6748	}
6749	}
6750
6751	if (fSupported)
6752	{
6753	uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
6754	switch (uWidth)
6755	{
6756	case VMX_VMCSFIELD_WIDTH_16BIT:
6757	{
6758	uint16_t u16Val;
6759	rc = VMXReadVmcs16(uVmcsField, &u16Val);
6760	AssertRC(rc);
6761	Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
6762
6763	if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
6764	&& uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
6765	{
6766	if (u16Val < HostGdtr.cbGdt)
6767	{
6768	/* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
6769	static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
6770	"Host FS", "Host GS", "Host TR" };
6771	uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
6772	Assert(idxSel < RT_ELEMENTS(s_apszSel));
6773	PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
6774	hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
6775	}
6776	else
6777	Log4((" Selector value exceeds GDT limit!\n"));
6778	}
6779	break;
6780	}
6781
6782	case VMX_VMCSFIELD_WIDTH_32BIT:
6783	{
6784	uint32_t u32Val;
6785	rc = VMXReadVmcs32(uVmcsField, &u32Val);
6786	AssertRC(rc);
6787	Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
6788	break;
6789	}
6790
6791	case VMX_VMCSFIELD_WIDTH_64BIT:
6792	case VMX_VMCSFIELD_WIDTH_NATURAL:
6793	{
6794	uint64_t u64Val;
6795	rc = VMXReadVmcs64(uVmcsField, &u64Val);
6796	AssertRC(rc);
6797	Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
6798	break;
6799	}
6800	}
6801	}
6802	}
6803
6804	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6805	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6806	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6807	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6808	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6809	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6810	#endif /* VBOX_STRICT */
6811	break;
6812	}
6813
6814	default:
6815	/* Impossible */
6816	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6817	break;
6818	}
6819	}
6820
6821
6822	/**
6823	* Sets up the usage of TSC-offsetting and updates the VMCS.
6824	*
6825	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6826	* VMX-preemption timer.
6827	*
6828	* @returns VBox status code.
6829	* @param pVCpu The cross context virtual CPU structure.
6830	* @param pVmxTransient The VMX-transient structure.
6831	*
6832	* @remarks No-long-jump zone!!!
6833	*/
6834	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6835	{
6836	bool fOffsettedTsc;
6837	bool fParavirtTsc;
6838	uint64_t uTscOffset;
6839	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6840	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6841
6842	if (pVM->hm.s.vmx.fUsePreemptTimer)
6843	{
6844	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6845
6846	/* Make sure the returned values have sane upper and lower boundaries. */
6847	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6848	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6849	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6850	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6851
6852	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6853	* preemption timers here. We probably need to clamp the preemption timer,
6854	* after converting the timer value to the host. */
6855	uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6856	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6857	AssertRC(rc);
6858	}
6859	else
6860	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6861
6862	if (fParavirtTsc)
6863	{
6864	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6865	information before every VM-entry, hence disable it for performance sake. */
6866	#if 0
6867	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6868	AssertRC(rc);
6869	#endif
6870	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6871	}
6872
6873	if ( fOffsettedTsc
6874	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6875	{
6876	if (pVmxTransient->fIsNestedGuest)
6877	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6878	hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
6879	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6880	}
6881	else
6882	{
6883	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6884	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6885	}
6886	}
6887
6888
6889	/**
6890	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6891	* VM-exit interruption info type.
6892	*
6893	* @returns The IEM exception flags.
6894	* @param uVector The event vector.
6895	* @param uVmxEventType The VMX event type.
6896	*
6897	* @remarks This function currently only constructs flags required for
6898	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6899	* and CR2 aspects of an exception are not included).
6900	*/
6901	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6902	{
6903	uint32_t fIemXcptFlags;
6904	switch (uVmxEventType)
6905	{
6906	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6907	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6908	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6909	break;
6910
6911	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6912	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6913	break;
6914
6915	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6916	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6917	break;
6918
6919	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6920	{
6921	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6922	if (uVector == X86_XCPT_BP)
6923	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6924	else if (uVector == X86_XCPT_OF)
6925	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6926	else
6927	{
6928	fIemXcptFlags = 0;
6929	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6930	}
6931	break;
6932	}
6933
6934	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6935	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6936	break;
6937
6938	default:
6939	fIemXcptFlags = 0;
6940	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6941	break;
6942	}
6943	return fIemXcptFlags;
6944	}
6945
6946
6947	/**
6948	* Sets an event as a pending event to be injected into the guest.
6949	*
6950	* @param pVCpu The cross context virtual CPU structure.
6951	* @param u32IntInfo The VM-entry interruption-information field.
6952	* @param cbInstr The VM-entry instruction length in bytes (for
6953	* software interrupts, exceptions and privileged
6954	* software exceptions).
6955	* @param u32ErrCode The VM-entry exception error code.
6956	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6957	* page-fault.
6958	*/
6959	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6960	RTGCUINTPTR GCPtrFaultAddress)
6961	{
6962	Assert(!pVCpu->hm.s.Event.fPending);
6963	pVCpu->hm.s.Event.fPending = true;
6964	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6965	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6966	pVCpu->hm.s.Event.cbInstr = cbInstr;
6967	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6968	}
6969
6970
6971	/**
6972	* Sets an external interrupt as pending-for-injection into the VM.
6973	*
6974	* @param pVCpu The cross context virtual CPU structure.
6975	* @param u8Interrupt The external interrupt vector.
6976	*/
6977	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
6978	{
6979	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6980	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6981	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6982	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6983	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6984	}
6985
6986
6987	/**
6988	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6989	*
6990	* @param pVCpu The cross context virtual CPU structure.
6991	*/
6992	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPUCC pVCpu)
6993	{
6994	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6995	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6996	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6997	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6998	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6999	}
7000
7001
7002	/**
7003	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7004	*
7005	* @param pVCpu The cross context virtual CPU structure.
7006	*/
7007	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPUCC pVCpu)
7008	{
7009	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7010	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7011	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7012	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7013	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7014	}
7015
7016
7017	/**
7018	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7019	*
7020	* @param pVCpu The cross context virtual CPU structure.
7021	*/
7022	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPUCC pVCpu)
7023	{
7024	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7025	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7026	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7027	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7028	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7029	}
7030
7031
7032	/**
7033	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7034	*
7035	* @param pVCpu The cross context virtual CPU structure.
7036	*/
7037	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPUCC pVCpu)
7038	{
7039	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7040	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7041	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7042	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7043	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7044	}
7045
7046
7047	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7048	/**
7049	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7050	*
7051	* @param pVCpu The cross context virtual CPU structure.
7052	* @param u32ErrCode The error code for the general-protection exception.
7053	*/
7054	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7055	{
7056	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7057	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7058	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7059	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7060	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7061	}
7062
7063
7064	/**
7065	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7066	*
7067	* @param pVCpu The cross context virtual CPU structure.
7068	* @param u32ErrCode The error code for the stack exception.
7069	*/
7070	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7071	{
7072	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7073	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7074	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7075	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7076	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7077	}
7078	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7079
7080
7081	/**
7082	* Fixes up attributes for the specified segment register.
7083	*
7084	* @param pVCpu The cross context virtual CPU structure.
7085	* @param pSelReg The segment register that needs fixing.
7086	* @param idxSel The VMCS field for the corresponding segment register.
7087	*/
7088	static void hmR0VmxFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7089	{
7090	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7091
7092	/*
7093	* If VT-x marks the segment as unusable, most other bits remain undefined:
7094	* - For CS the L, D and G bits have meaning.
7095	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7096	* - For the remaining data segments no bits are defined.
7097	*
7098	* The present bit and the unusable bit has been observed to be set at the
7099	* same time (the selector was supposed to be invalid as we started executing
7100	* a V8086 interrupt in ring-0).
7101	*
7102	* What should be important for the rest of the VBox code, is that the P bit is
7103	* cleared. Some of the other VBox code recognizes the unusable bit, but
7104	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7105	* safe side here, we'll strip off P and other bits we don't care about. If
7106	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7107	*
7108	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7109	*/
7110	#ifdef VBOX_STRICT
7111	uint32_t const uAttr = pSelReg->Attr.u;
7112	#endif
7113
7114	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7115	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7116	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7117
7118	#ifdef VBOX_STRICT
7119	VMMRZCallRing3Disable(pVCpu);
7120	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7121	# ifdef DEBUG_bird
7122	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7123	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7124	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7125	# endif
7126	VMMRZCallRing3Enable(pVCpu);
7127	NOREF(uAttr);
7128	#endif
7129	RT_NOREF2(pVCpu, idxSel);
7130	}
7131
7132
7133	/**
7134	* Imports a guest segment register from the current VMCS into the guest-CPU
7135	* context.
7136	*
7137	* @param pVCpu The cross context virtual CPU structure.
7138	* @param iSegReg The segment register number (X86_SREG_XXX).
7139	*
7140	* @remarks Called with interrupts and/or preemption disabled.
7141	*/
7142	static void hmR0VmxImportGuestSegReg(PVMCPUCC pVCpu, uint8_t iSegReg)
7143	{
7144	Assert(iSegReg < X86_SREG_COUNT);
7145
7146	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7147	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7148	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7149	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7150
7151	uint16_t u16Sel;
7152	uint64_t u64Base;
7153	uint32_t u32Limit, u32Attr;
7154	int rc = VMXReadVmcs16(idxSel, &u16Sel); AssertRC(rc);
7155	rc = VMXReadVmcs32(idxLimit, &u32Limit); AssertRC(rc);
7156	rc = VMXReadVmcs32(idxAttr, &u32Attr); AssertRC(rc);
7157	rc = VMXReadVmcsNw(idxBase, &u64Base); AssertRC(rc);
7158
7159	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7160	pSelReg->Sel = u16Sel;
7161	pSelReg->ValidSel = u16Sel;
7162	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7163	pSelReg->u32Limit = u32Limit;
7164	pSelReg->u64Base = u64Base;
7165	pSelReg->Attr.u = u32Attr;
7166	if (u32Attr & X86DESCATTR_UNUSABLE)
7167	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7168	}
7169
7170
7171	/**
7172	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7173	*
7174	* @param pVCpu The cross context virtual CPU structure.
7175	*
7176	* @remarks Called with interrupts and/or preemption disabled.
7177	*/
7178	static void hmR0VmxImportGuestLdtr(PVMCPUCC pVCpu)
7179	{
7180	uint16_t u16Sel;
7181	uint64_t u64Base;
7182	uint32_t u32Limit, u32Attr;
7183	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
7184	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
7185	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7186	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
7187
7188	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
7189	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
7190	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7191	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7192	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7193	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7194	if (u32Attr & X86DESCATTR_UNUSABLE)
7195	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7196	}
7197
7198
7199	/**
7200	* Imports the guest TR from the current VMCS into the guest-CPU context.
7201	*
7202	* @param pVCpu The cross context virtual CPU structure.
7203	*
7204	* @remarks Called with interrupts and/or preemption disabled.
7205	*/
7206	static void hmR0VmxImportGuestTr(PVMCPUCC pVCpu)
7207	{
7208	uint16_t u16Sel;
7209	uint64_t u64Base;
7210	uint32_t u32Limit, u32Attr;
7211	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
7212	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
7213	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7214	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
7215
7216	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
7217	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
7218	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7219	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7220	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7221	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7222	/* TR is the only selector that can never be unusable. */
7223	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7224	}
7225
7226
7227	/**
7228	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7229	*
7230	* @param pVCpu The cross context virtual CPU structure.
7231	*
7232	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7233	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7234	* instead!!!
7235	*/
7236	static void hmR0VmxImportGuestRip(PVMCPUCC pVCpu)
7237	{
7238	uint64_t u64Val;
7239	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7240	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7241	{
7242	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
7243	AssertRC(rc);
7244
7245	pCtx->rip = u64Val;
7246	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7247	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7248	}
7249	}
7250
7251
7252	/**
7253	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7254	*
7255	* @param pVCpu The cross context virtual CPU structure.
7256	* @param pVmcsInfo The VMCS info. object.
7257	*
7258	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7259	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7260	* instead!!!
7261	*/
7262	static void hmR0VmxImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7263	{
7264	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7265	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7266	{
7267	uint64_t u64Val;
7268	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7269	AssertRC(rc);
7270
7271	pCtx->rflags.u64 = u64Val;
7272	if (pVmcsInfo->RealMode.fRealOnV86Active)
7273	{
7274	pCtx->eflags.Bits.u1VM = 0;
7275	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7276	}
7277	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7278	}
7279	}
7280
7281
7282	/**
7283	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7284	* context.
7285	*
7286	* @param pVCpu The cross context virtual CPU structure.
7287	* @param pVmcsInfo The VMCS info. object.
7288	*
7289	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7290	* do not log!
7291	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7292	* instead!!!
7293	*/
7294	static void hmR0VmxImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7295	{
7296	uint32_t u32Val;
7297	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
7298	if (!u32Val)
7299	{
7300	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7301	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7302	CPUMSetGuestNmiBlocking(pVCpu, false);
7303	}
7304	else
7305	{
7306	/*
7307	* We must import RIP here to set our EM interrupt-inhibited state.
7308	* We also import RFLAGS as our code that evaluates pending interrupts
7309	* before VM-entry requires it.
7310	*/
7311	hmR0VmxImportGuestRip(pVCpu);
7312	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7313
7314	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7315	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7316	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7317	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7318
7319	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7320	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7321	}
7322	}
7323
7324
7325	/**
7326	* Worker for VMXR0ImportStateOnDemand.
7327	*
7328	* @returns VBox status code.
7329	* @param pVCpu The cross context virtual CPU structure.
7330	* @param pVmcsInfo The VMCS info. object.
7331	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7332	*/
7333	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7334	{
7335	int rc = VINF_SUCCESS;
7336	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7337	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7338	uint32_t u32Val;
7339
7340	/*
7341	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7342	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7343	* neither are other host platforms.
7344	*
7345	* Committing this temporarily as it prevents BSOD.
7346	*
7347	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7348	*/
7349	#ifdef RT_OS_WINDOWS
7350	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7351	return VERR_HM_IPE_1;
7352	#endif
7353
7354	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7355
7356	/*
7357	* We disable interrupts to make the updating of the state and in particular
7358	* the fExtrn modification atomic wrt to preemption hooks.
7359	*/
7360	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7361
7362	fWhat &= pCtx->fExtrn;
7363	if (fWhat)
7364	{
7365	do
7366	{
7367	if (fWhat & CPUMCTX_EXTRN_RIP)
7368	hmR0VmxImportGuestRip(pVCpu);
7369
7370	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7371	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7372
7373	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7374	hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7375
7376	if (fWhat & CPUMCTX_EXTRN_RSP)
7377	{
7378	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &pCtx->rsp);
7379	AssertRC(rc);
7380	}
7381
7382	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7383	{
7384	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7385	if (fWhat & CPUMCTX_EXTRN_CS)
7386	{
7387	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7388	hmR0VmxImportGuestRip(pVCpu);
7389	if (fRealOnV86Active)
7390	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7391	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7392	}
7393	if (fWhat & CPUMCTX_EXTRN_SS)
7394	{
7395	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7396	if (fRealOnV86Active)
7397	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7398	}
7399	if (fWhat & CPUMCTX_EXTRN_DS)
7400	{
7401	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7402	if (fRealOnV86Active)
7403	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7404	}
7405	if (fWhat & CPUMCTX_EXTRN_ES)
7406	{
7407	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7408	if (fRealOnV86Active)
7409	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7410	}
7411	if (fWhat & CPUMCTX_EXTRN_FS)
7412	{
7413	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7414	if (fRealOnV86Active)
7415	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7416	}
7417	if (fWhat & CPUMCTX_EXTRN_GS)
7418	{
7419	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7420	if (fRealOnV86Active)
7421	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7422	}
7423	}
7424
7425	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7426	{
7427	if (fWhat & CPUMCTX_EXTRN_LDTR)
7428	hmR0VmxImportGuestLdtr(pVCpu);
7429
7430	if (fWhat & CPUMCTX_EXTRN_GDTR)
7431	{
7432	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
7433	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
7434	pCtx->gdtr.cbGdt = u32Val;
7435	}
7436
7437	/* Guest IDTR. */
7438	if (fWhat & CPUMCTX_EXTRN_IDTR)
7439	{
7440	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
7441	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
7442	pCtx->idtr.cbIdt = u32Val;
7443	}
7444
7445	/* Guest TR. */
7446	if (fWhat & CPUMCTX_EXTRN_TR)
7447	{
7448	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7449	don't need to import that one. */
7450	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7451	hmR0VmxImportGuestTr(pVCpu);
7452	}
7453	}
7454
7455	if (fWhat & CPUMCTX_EXTRN_DR7)
7456	{
7457	if (!pVCpu->hm.s.fUsingHyperDR7)
7458	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &pCtx->dr[7]); AssertRC(rc);
7459	}
7460
7461	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7462	{
7463	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
7464	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
7465	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
7466	pCtx->SysEnter.cs = u32Val;
7467	}
7468
7469	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7470	{
7471	if ( pVM->hm.s.fAllow64BitGuests
7472	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7473	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7474	}
7475
7476	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7477	{
7478	if ( pVM->hm.s.fAllow64BitGuests
7479	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7480	{
7481	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7482	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7483	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7484	}
7485	}
7486
7487	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7488	{
7489	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7490	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7491	Assert(pMsrs);
7492	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7493	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7494	for (uint32_t i = 0; i < cMsrs; i++)
7495	{
7496	uint32_t const idMsr = pMsrs[i].u32Msr;
7497	switch (idMsr)
7498	{
7499	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7500	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7501	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7502	default:
7503	{
7504	pCtx->fExtrn = 0;
7505	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7506	ASMSetFlags(fEFlags);
7507	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7508	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7509	}
7510	}
7511	}
7512	}
7513
7514	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7515	{
7516	if (fWhat & CPUMCTX_EXTRN_CR0)
7517	{
7518	uint64_t u64Cr0;
7519	uint64_t u64Shadow;
7520	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
7521	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
7522	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7523	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7524	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7525	#else
7526	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7527	{
7528	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7529	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7530	}
7531	else
7532	{
7533	/*
7534	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
7535	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7536	* re-construct CR0. See @bugref{9180#c95} for details.
7537	*/
7538	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7539	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7540	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7541	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
7542	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
7543	}
7544	#endif
7545	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7546	CPUMSetGuestCR0(pVCpu, u64Cr0);
7547	VMMRZCallRing3Enable(pVCpu);
7548	}
7549
7550	if (fWhat & CPUMCTX_EXTRN_CR4)
7551	{
7552	uint64_t u64Cr4;
7553	uint64_t u64Shadow;
7554	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
7555	rc \|= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
7556	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7557	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7558	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7559	#else
7560	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7561	{
7562	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7563	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7564	}
7565	else
7566	{
7567	/*
7568	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
7569	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7570	* re-construct CR4. See @bugref{9180#c95} for details.
7571	*/
7572	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7573	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7574	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7575	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
7576	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
7577	}
7578	#endif
7579	pCtx->cr4 = u64Cr4;
7580	}
7581
7582	if (fWhat & CPUMCTX_EXTRN_CR3)
7583	{
7584	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7585	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7586	\|\| ( pVM->hm.s.fNestedPaging
7587	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7588	{
7589	uint64_t u64Cr3;
7590	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
7591	if (pCtx->cr3 != u64Cr3)
7592	{
7593	pCtx->cr3 = u64Cr3;
7594	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7595	}
7596
7597	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7598	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7599	if (CPUMIsGuestInPAEModeEx(pCtx))
7600	{
7601	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
7602	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
7603	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
7604	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
7605	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7606	}
7607	}
7608	}
7609	}
7610
7611	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7612	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7613	{
7614	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7615	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7616	{
7617	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7618	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7619	if (RT_SUCCESS(rc))
7620	{ /* likely */ }
7621	else
7622	break;
7623	}
7624	}
7625	#endif
7626	} while (0);
7627
7628	if (RT_SUCCESS(rc))
7629	{
7630	/* Update fExtrn. */
7631	pCtx->fExtrn &= ~fWhat;
7632
7633	/* If everything has been imported, clear the HM keeper bit. */
7634	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7635	{
7636	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7637	Assert(!pCtx->fExtrn);
7638	}
7639	}
7640	}
7641	else
7642	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7643
7644	/*
7645	* Restore interrupts.
7646	*/
7647	ASMSetFlags(fEFlags);
7648
7649	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7650
7651	if (RT_SUCCESS(rc))
7652	{ /* likely */ }
7653	else
7654	return rc;
7655
7656	/*
7657	* Honor any pending CR3 updates.
7658	*
7659	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
7660	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7661	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7662	*
7663	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7664	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7665	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7666	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7667	*
7668	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7669	*/
7670	if (VMMRZCallRing3IsEnabled(pVCpu))
7671	{
7672	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7673	{
7674	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7675	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7676	}
7677
7678	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7679	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7680
7681	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7682	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7683	}
7684
7685	return VINF_SUCCESS;
7686	}
7687
7688
7689	/**
7690	* Saves the guest state from the VMCS into the guest-CPU context.
7691	*
7692	* @returns VBox status code.
7693	* @param pVCpu The cross context virtual CPU structure.
7694	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7695	*/
7696	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
7697	{
7698	AssertPtr(pVCpu);
7699	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7700	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7701	}
7702
7703
7704	/**
7705	* Check per-VM and per-VCPU force flag actions that require us to go back to
7706	* ring-3 for one reason or another.
7707	*
7708	* @returns Strict VBox status code (i.e. informational status codes too)
7709	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7710	* ring-3.
7711	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7712	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7713	* interrupts)
7714	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7715	* all EMTs to be in ring-3.
7716	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7717	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7718	* to the EM loop.
7719	*
7720	* @param pVCpu The cross context virtual CPU structure.
7721	* @param fStepping Whether we are single-stepping the guest using the
7722	* hypervisor debugger.
7723	*
7724	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7725	* is no longer in VMX non-root mode.
7726	*/
7727	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPUCC pVCpu, bool fStepping)
7728	{
7729	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7730
7731	/*
7732	* Update pending interrupts into the APIC's IRR.
7733	*/
7734	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7735	APICUpdatePendingInterrupts(pVCpu);
7736
7737	/*
7738	* Anything pending? Should be more likely than not if we're doing a good job.
7739	*/
7740	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7741	if ( !fStepping
7742	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7743	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7744	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7745	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7746	return VINF_SUCCESS;
7747
7748	/* Pending PGM C3 sync. */
7749	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7750	{
7751	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7752	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7753	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7754	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7755	if (rcStrict != VINF_SUCCESS)
7756	{
7757	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7758	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7759	return rcStrict;
7760	}
7761	}
7762
7763	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7764	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7765	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7766	{
7767	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7768	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7769	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7770	return rc;
7771	}
7772
7773	/* Pending VM request packets, such as hardware interrupts. */
7774	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7775	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7776	{
7777	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7778	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7779	return VINF_EM_PENDING_REQUEST;
7780	}
7781
7782	/* Pending PGM pool flushes. */
7783	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7784	{
7785	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7786	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7787	return VINF_PGM_POOL_FLUSH_PENDING;
7788	}
7789
7790	/* Pending DMA requests. */
7791	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7792	{
7793	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7794	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7795	return VINF_EM_RAW_TO_R3;
7796	}
7797
7798	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7799	/* Pending nested-guest APIC-write (has highest priority among nested-guest FFs). */
7800	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7801	{
7802	Log4Func(("Pending nested-guest APIC-write\n"));
7803	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
7804	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7805	return rcStrict;
7806	}
7807	/** @todo VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_PREEMPT_TIMER */
7808	#endif
7809
7810	return VINF_SUCCESS;
7811	}
7812
7813
7814	/**
7815	* Converts any TRPM trap into a pending HM event. This is typically used when
7816	* entering from ring-3 (not longjmp returns).
7817	*
7818	* @param pVCpu The cross context virtual CPU structure.
7819	*/
7820	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
7821	{
7822	Assert(TRPMHasTrap(pVCpu));
7823	Assert(!pVCpu->hm.s.Event.fPending);
7824
7825	uint8_t uVector;
7826	TRPMEVENT enmTrpmEvent;
7827	uint32_t uErrCode;
7828	RTGCUINTPTR GCPtrFaultAddress;
7829	uint8_t cbInstr;
7830	bool fIcebp;
7831
7832	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
7833	AssertRC(rc);
7834
7835	uint32_t u32IntInfo;
7836	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
7837	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
7838
7839	rc = TRPMResetTrap(pVCpu);
7840	AssertRC(rc);
7841	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7842	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7843
7844	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7845	}
7846
7847
7848	/**
7849	* Converts the pending HM event into a TRPM trap.
7850	*
7851	* @param pVCpu The cross context virtual CPU structure.
7852	*/
7853	static void hmR0VmxPendingEventToTrpmTrap(PVMCPUCC pVCpu)
7854	{
7855	Assert(pVCpu->hm.s.Event.fPending);
7856
7857	/* If a trap was already pending, we did something wrong! */
7858	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7859
7860	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
7861	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
7862	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
7863
7864	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7865
7866	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7867	AssertRC(rc);
7868
7869	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7870	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
7871
7872	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
7873	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7874	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
7875	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7876
7877	if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7878	TRPMSetTrapDueToIcebp(pVCpu);
7879
7880	/* We're now done converting the pending event. */
7881	pVCpu->hm.s.Event.fPending = false;
7882	}
7883
7884
7885	/**
7886	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7887	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7888	*
7889	* @param pVCpu The cross context virtual CPU structure.
7890	* @param pVmcsInfo The VMCS info. object.
7891	*/
7892	static void hmR0VmxSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7893	{
7894	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7895	{
7896	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7897	{
7898	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7899	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7900	AssertRC(rc);
7901	}
7902	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7903	}
7904
7905
7906	/**
7907	* Clears the interrupt-window exiting control in the VMCS.
7908	*
7909	* @param pVmcsInfo The VMCS info. object.
7910	*/
7911	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7912	{
7913	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7914	{
7915	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7916	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7917	AssertRC(rc);
7918	}
7919	}
7920
7921
7922	/**
7923	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7924	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7925	*
7926	* @param pVCpu The cross context virtual CPU structure.
7927	* @param pVmcsInfo The VMCS info. object.
7928	*/
7929	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7930	{
7931	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7932	{
7933	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7934	{
7935	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7936	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7937	AssertRC(rc);
7938	Log4Func(("Setup NMI-window exiting\n"));
7939	}
7940	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7941	}
7942
7943
7944	/**
7945	* Clears the NMI-window exiting control in the VMCS.
7946	*
7947	* @param pVmcsInfo The VMCS info. object.
7948	*/
7949	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7950	{
7951	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7952	{
7953	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7954	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7955	AssertRC(rc);
7956	}
7957	}
7958
7959
7960	/**
7961	* Does the necessary state syncing before returning to ring-3 for any reason
7962	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7963	*
7964	* @returns VBox status code.
7965	* @param pVCpu The cross context virtual CPU structure.
7966	* @param fImportState Whether to import the guest state from the VMCS back
7967	* to the guest-CPU context.
7968	*
7969	* @remarks No-long-jmp zone!!!
7970	*/
7971	static int hmR0VmxLeave(PVMCPUCC pVCpu, bool fImportState)
7972	{
7973	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7974	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7975
7976	RTCPUID const idCpu = RTMpCpuId();
7977	Log4Func(("HostCpuId=%u\n", idCpu));
7978
7979	/*
7980	* !!! IMPORTANT !!!
7981	* If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
7982	*/
7983
7984	/* Save the guest state if necessary. */
7985	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7986	if (fImportState)
7987	{
7988	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
7989	AssertRCReturn(rc, rc);
7990	}
7991
7992	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7993	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7994	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7995
7996	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7997	#ifdef VBOX_STRICT
7998	if (CPUMIsHyperDebugStateActive(pVCpu))
7999	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8000	#endif
8001	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8002	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8003	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8004
8005	/* Restore host-state bits that VT-x only restores partially. */
8006	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8007	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8008	{
8009	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8010	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8011	}
8012	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8013
8014	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8015	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8016	{
8017	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8018	if (!fImportState)
8019	{
8020	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8021	AssertRCReturn(rc, rc);
8022	}
8023	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8024	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8025	}
8026	else
8027	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8028
8029	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8030	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8031
8032	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8033	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8034	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8035	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8036	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8037	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8038	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8039	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8040	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8041	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8042
8043	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8044
8045	/** @todo This partially defeats the purpose of having preemption hooks.
8046	* The problem is, deregistering the hooks should be moved to a place that
8047	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8048	* context.
8049	*/
8050	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8051	AssertRCReturn(rc, rc);
8052
8053	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8054	/*
8055	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8056	* clear a shadow VMCS before allowing that VMCS to become active on another
8057	* logical processor. We may or may not be importing guest state which clears
8058	* it, so cover for it here.
8059	*
8060	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8061	*/
8062	if ( pVmcsInfo->pvShadowVmcs
8063	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8064	{
8065	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8066	AssertRCReturn(rc, rc);
8067	}
8068
8069	/*
8070	* Flag that we need to re-export the host state if we switch to this VMCS before
8071	* executing guest or nested-guest code.
8072	*/
8073	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
8074	#endif
8075
8076	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8077	NOREF(idCpu);
8078	return VINF_SUCCESS;
8079	}
8080
8081
8082	/**
8083	* Leaves the VT-x session.
8084	*
8085	* @returns VBox status code.
8086	* @param pVCpu The cross context virtual CPU structure.
8087	*
8088	* @remarks No-long-jmp zone!!!
8089	*/
8090	static int hmR0VmxLeaveSession(PVMCPUCC pVCpu)
8091	{
8092	HM_DISABLE_PREEMPT(pVCpu);
8093	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8094	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8095	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8096
8097	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8098	and done this from the VMXR0ThreadCtxCallback(). */
8099	if (!pVCpu->hm.s.fLeaveDone)
8100	{
8101	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8102	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8103	pVCpu->hm.s.fLeaveDone = true;
8104	}
8105	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8106
8107	/*
8108	* !!! IMPORTANT !!!
8109	* If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
8110	*/
8111
8112	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8113	/** @todo Deregistering here means we need to VMCLEAR always
8114	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8115	* for calling VMMR0ThreadCtxHookDisable here! */
8116	VMMR0ThreadCtxHookDisable(pVCpu);
8117
8118	/* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
8119	int rc = HMR0LeaveCpu(pVCpu);
8120	HM_RESTORE_PREEMPT();
8121	return rc;
8122	}
8123
8124
8125	/**
8126	* Does the necessary state syncing before doing a longjmp to ring-3.
8127	*
8128	* @returns VBox status code.
8129	* @param pVCpu The cross context virtual CPU structure.
8130	*
8131	* @remarks No-long-jmp zone!!!
8132	*/
8133	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPUCC pVCpu)
8134	{
8135	return hmR0VmxLeaveSession(pVCpu);
8136	}
8137
8138
8139	/**
8140	* Take necessary actions before going back to ring-3.
8141	*
8142	* An action requires us to go back to ring-3. This function does the necessary
8143	* steps before we can safely return to ring-3. This is not the same as longjmps
8144	* to ring-3, this is voluntary and prepares the guest so it may continue
8145	* executing outside HM (recompiler/IEM).
8146	*
8147	* @returns VBox status code.
8148	* @param pVCpu The cross context virtual CPU structure.
8149	* @param rcExit The reason for exiting to ring-3. Can be
8150	* VINF_VMM_UNKNOWN_RING3_CALL.
8151	*/
8152	static int hmR0VmxExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
8153	{
8154	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8155
8156	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8157	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8158	{
8159	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8160	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8161	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8162	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8163	}
8164
8165	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8166	VMMRZCallRing3Disable(pVCpu);
8167	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8168
8169	/*
8170	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8171	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8172	*
8173	* This is because execution may continue from ring-3 and we would need to inject
8174	* the event from there (hence place it back in TRPM).
8175	*/
8176	if (pVCpu->hm.s.Event.fPending)
8177	{
8178	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8179	Assert(!pVCpu->hm.s.Event.fPending);
8180
8181	/* Clear the events from the VMCS. */
8182	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8183	AssertRC(rc);
8184	}
8185	#ifdef VBOX_STRICT
8186	else
8187	{
8188	/*
8189	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8190	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8191	* occasionally, see @bugref{9180#c42}.
8192	*
8193	* However, if the VM-entry failed, any VM entry-interruption info. field would
8194	* be left unmodified as the event would not have been injected to the guest. In
8195	* such cases, don't assert, we're not going to continue guest execution anyway.
8196	*/
8197	uint32_t uExitReason;
8198	uint32_t uEntryIntInfo;
8199	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8200	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8201	AssertRC(rc);
8202	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8203	}
8204	#endif
8205
8206	/*
8207	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8208	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8209	* (e.g. TPR below threshold).
8210	*/
8211	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8212	{
8213	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8214	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8215	}
8216
8217	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8218	and if we're injecting an event we should have a TRPM trap pending. */
8219	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8220	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8221	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8222	#endif
8223
8224	/* Save guest state and restore host state bits. */
8225	int rc = hmR0VmxLeaveSession(pVCpu);
8226	AssertRCReturn(rc, rc);
8227	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8228
8229	/* Thread-context hooks are unregistered at this point!!! */
8230	/* Ring-3 callback notifications are unregistered at this point!!! */
8231
8232	/* Sync recompiler state. */
8233	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8234	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8235	\| CPUM_CHANGED_LDTR
8236	\| CPUM_CHANGED_GDTR
8237	\| CPUM_CHANGED_IDTR
8238	\| CPUM_CHANGED_TR
8239	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8240	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8241	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8242	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8243
8244	Assert(!pVCpu->hm.s.fClearTrapFlag);
8245
8246	/* Update the exit-to-ring 3 reason. */
8247	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8248
8249	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8250	if ( rcExit != VINF_EM_RAW_INTERRUPT
8251	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8252	{
8253	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8254	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8255	}
8256
8257	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8258	VMMRZCallRing3Enable(pVCpu);
8259	return rc;
8260	}
8261
8262
8263	/**
8264	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8265	* longjump to ring-3 and possibly get preempted.
8266	*
8267	* @returns VBox status code.
8268	* @param pVCpu The cross context virtual CPU structure.
8269	* @param enmOperation The operation causing the ring-3 longjump.
8270	*/
8271	VMMR0DECL(int) VMXR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
8272	{
8273	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8274	{
8275	/*
8276	* !!! IMPORTANT !!!
8277	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8278	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8279	*/
8280	VMMRZCallRing3RemoveNotification(pVCpu);
8281	VMMRZCallRing3Disable(pVCpu);
8282	HM_DISABLE_PREEMPT(pVCpu);
8283
8284	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8285	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8286	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8287	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8288
8289	/* Restore host-state bits that VT-x only restores partially. */
8290	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8291	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8292	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8293	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8294
8295	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8296	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8297	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8298
8299	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8300	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8301	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8302
8303	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8304	cleared as part of importing the guest state above. */
8305	hmR0VmxClearVmcs(pVmcsInfo);
8306
8307	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8308	VMMR0ThreadCtxHookDisable(pVCpu);
8309
8310	/* Leave HM context. This takes care of local init (term). */
8311	HMR0LeaveCpu(pVCpu);
8312	HM_RESTORE_PREEMPT();
8313	return VINF_SUCCESS;
8314	}
8315
8316	Assert(pVCpu);
8317	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8318	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8319
8320	VMMRZCallRing3Disable(pVCpu);
8321	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8322
8323	Log4Func(("-> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8324
8325	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8326	AssertRCReturn(rc, rc);
8327
8328	VMMRZCallRing3Enable(pVCpu);
8329	return VINF_SUCCESS;
8330	}
8331
8332
8333	/**
8334	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8335	* stack.
8336	*
8337	* @returns Strict VBox status code (i.e. informational status codes too).
8338	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8339	* @param pVCpu The cross context virtual CPU structure.
8340	* @param uValue The value to push to the guest stack.
8341	*/
8342	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
8343	{
8344	/*
8345	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8346	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8347	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8348	*/
8349	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8350	if (pCtx->sp == 1)
8351	return VINF_EM_RESET;
8352	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8353	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8354	AssertRC(rc);
8355	return rc;
8356	}
8357
8358
8359	/**
8360	* Injects an event into the guest upon VM-entry by updating the relevant fields
8361	* in the VM-entry area in the VMCS.
8362	*
8363	* @returns Strict VBox status code (i.e. informational status codes too).
8364	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8365	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8366	*
8367	* @param pVCpu The cross context virtual CPU structure.
8368	* @param pVmxTransient The VMX-transient structure.
8369	* @param pEvent The event being injected.
8370	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8371	* will be updated if necessary. This cannot not be NULL.
8372	* @param fStepping Whether we're single-stepping guest execution and should
8373	* return VINF_EM_DBG_STEPPED if the event is injected
8374	* directly (registers modified by us, not by hardware on
8375	* VM-entry).
8376	*/
8377	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8378	uint32_t *pfIntrState)
8379	{
8380	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8381	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8382	Assert(pfIntrState);
8383
8384	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8385	uint32_t u32IntInfo = pEvent->u64IntInfo;
8386	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8387	uint32_t const cbInstr = pEvent->cbInstr;
8388	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8389	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8390	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8391
8392	#ifdef VBOX_STRICT
8393	/*
8394	* Validate the error-code-valid bit for hardware exceptions.
8395	* No error codes for exceptions in real-mode.
8396	*
8397	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8398	*/
8399	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8400	&& !CPUMIsGuestInRealModeEx(pCtx))
8401	{
8402	switch (uVector)
8403	{
8404	case X86_XCPT_PF:
8405	case X86_XCPT_DF:
8406	case X86_XCPT_TS:
8407	case X86_XCPT_NP:
8408	case X86_XCPT_SS:
8409	case X86_XCPT_GP:
8410	case X86_XCPT_AC:
8411	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8412	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8413	RT_FALL_THRU();
8414	default:
8415	break;
8416	}
8417	}
8418
8419	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8420	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8421	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8422	#endif
8423
8424	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8425
8426	/*
8427	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8428	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8429	* interrupt handler in the (real-mode) guest.
8430	*
8431	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8432	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8433	*/
8434	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8435	{
8436	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8437	{
8438	/*
8439	* For CPUs with unrestricted guest execution enabled and with the guest
8440	* in real-mode, we must not set the deliver-error-code bit.
8441	*
8442	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8443	*/
8444	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8445	}
8446	else
8447	{
8448	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8449	Assert(PDMVmmDevHeapIsEnabled(pVM));
8450	Assert(pVM->hm.s.vmx.pRealModeTSS);
8451	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8452
8453	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8454	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8455	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8456	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8457	AssertRCReturn(rc2, rc2);
8458
8459	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8460	size_t const cbIdtEntry = sizeof(X86IDTR16);
8461	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8462	{
8463	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8464	if (uVector == X86_XCPT_DF)
8465	return VINF_EM_RESET;
8466
8467	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8468	No error codes for exceptions in real-mode. */
8469	if (uVector == X86_XCPT_GP)
8470	{
8471	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8472	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8473	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8474	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8475	HMEVENT EventXcptDf;
8476	RT_ZERO(EventXcptDf);
8477	EventXcptDf.u64IntInfo = uXcptDfInfo;
8478	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8479	}
8480
8481	/*
8482	* If we're injecting an event with no valid IDT entry, inject a #GP.
8483	* No error codes for exceptions in real-mode.
8484	*
8485	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8486	*/
8487	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8488	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8489	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8490	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8491	HMEVENT EventXcptGp;
8492	RT_ZERO(EventXcptGp);
8493	EventXcptGp.u64IntInfo = uXcptGpInfo;
8494	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8495	}
8496
8497	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8498	uint16_t uGuestIp = pCtx->ip;
8499	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8500	{
8501	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8502	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8503	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8504	}
8505	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8506	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8507
8508	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8509	X86IDTR16 IdtEntry;
8510	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8511	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8512	AssertRCReturn(rc2, rc2);
8513
8514	/* Construct the stack frame for the interrupt/exception handler. */
8515	VBOXSTRICTRC rcStrict;
8516	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8517	if (rcStrict == VINF_SUCCESS)
8518	{
8519	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8520	if (rcStrict == VINF_SUCCESS)
8521	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8522	}
8523
8524	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8525	if (rcStrict == VINF_SUCCESS)
8526	{
8527	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8528	pCtx->rip = IdtEntry.offSel;
8529	pCtx->cs.Sel = IdtEntry.uSel;
8530	pCtx->cs.ValidSel = IdtEntry.uSel;
8531	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8532	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8533	&& uVector == X86_XCPT_PF)
8534	pCtx->cr2 = GCPtrFault;
8535
8536	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8537	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8538	\| HM_CHANGED_GUEST_RSP);
8539
8540	/*
8541	* If we delivered a hardware exception (other than an NMI) and if there was
8542	* block-by-STI in effect, we should clear it.
8543	*/
8544	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8545	{
8546	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8547	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8548	Log4Func(("Clearing inhibition due to STI\n"));
8549	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8550	}
8551
8552	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8553	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8554
8555	/*
8556	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8557	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8558	*/
8559	pVCpu->hm.s.Event.fPending = false;
8560
8561	/*
8562	* If we eventually support nested-guest execution without unrestricted guest execution,
8563	* we should set fInterceptEvents here.
8564	*/
8565	Assert(!pVmxTransient->fIsNestedGuest);
8566
8567	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8568	if (fStepping)
8569	rcStrict = VINF_EM_DBG_STEPPED;
8570	}
8571	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8572	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8573	return rcStrict;
8574	}
8575	}
8576
8577	/*
8578	* Validate.
8579	*/
8580	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8581	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8582
8583	/*
8584	* Inject the event into the VMCS.
8585	*/
8586	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8587	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8588	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8589	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8590	AssertRC(rc);
8591
8592	/*
8593	* Update guest CR2 if this is a page-fault.
8594	*/
8595	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8596	pCtx->cr2 = GCPtrFault;
8597
8598	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8599	return VINF_SUCCESS;
8600	}
8601
8602
8603	/**
8604	* Evaluates the event to be delivered to the guest and sets it as the pending
8605	* event.
8606	*
8607	* @returns Strict VBox status code (i.e. informational status codes too).
8608	* @param pVCpu The cross context virtual CPU structure.
8609	* @param pVmxTransient The VMX-transient structure.
8610	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8611	*/
8612	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8613	{
8614	Assert(pfIntrState);
8615	Assert(!TRPMHasTrap(pVCpu));
8616
8617	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8618	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8619	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8620
8621	/*
8622	* Get the current interruptibility-state of the guest or nested-guest and
8623	* then figure out what needs to be injected.
8624	*/
8625	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8626	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8627	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8628	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8629
8630	/* We don't support block-by-SMI yet.*/
8631	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
8632
8633	/* Block-by-STI must not be set when interrupts are disabled. */
8634	if (fBlockSti)
8635	{
8636	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
8637	Assert(pCtx->eflags.Bits.u1IF);
8638	}
8639
8640	/* Update interruptibility state to the caller. */
8641	*pfIntrState = fIntrState;
8642
8643	/*
8644	* Toggling of interrupt force-flags here is safe since we update TRPM on
8645	* premature exits to ring-3 before executing guest code, see hmR0VmxExitToRing3().
8646	* We must NOT restore these force-flags.
8647	*/
8648
8649	/** @todo SMI. SMIs take priority over NMIs. */
8650
8651	/*
8652	* Check if an NMI is pending and if the guest or nested-guest can receive them.
8653	* NMIs take priority over external interrupts.
8654	*/
8655	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8656	{
8657	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8658	if ( !pVCpu->hm.s.Event.fPending
8659	&& !fBlockNmi
8660	&& !fBlockSti
8661	&& !fBlockMovSS)
8662	{
8663	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8664	if ( fIsNestedGuest
8665	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8666	return IEMExecVmxVmexitXcptNmi(pVCpu);
8667	#endif
8668	hmR0VmxSetPendingXcptNmi(pVCpu);
8669	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8670	Log4Func(("Pending NMI\n"));
8671	}
8672	else if (!fIsNestedGuest)
8673	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8674	/* else: for nested-guests, NMI-window exiting will be picked up when merging VMCS controls. */
8675	}
8676	/*
8677	* Check if an external interrupt (PIC/APIC) is pending and if the guest or nested-guest
8678	* can receive them. Once PDMGetInterrupt() returns a valid interrupt we -must- deliver
8679	* the interrupt. We can no longer re-request it from the APIC.
8680	*/
8681	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8682	&& !pVCpu->hm.s.fSingleInstruction)
8683	{
8684	Assert(!DBGFIsStepping(pVCpu));
8685	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8686	AssertRCReturn(rc, rc);
8687
8688	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8689	if ( !pVCpu->hm.s.Event.fPending
8690	&& !fBlockInt
8691	&& !fBlockSti
8692	&& !fBlockMovSS)
8693	{
8694	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8695	if ( fIsNestedGuest
8696	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8697	&& !CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8698	{
8699	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8700	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8701	return rcStrict;
8702	}
8703	#endif
8704	uint8_t u8Interrupt;
8705	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8706	if (RT_SUCCESS(rc))
8707	{
8708	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8709	if ( fIsNestedGuest
8710	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8711	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8712	{
8713	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8714	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8715	return rcStrict;
8716	}
8717	#endif
8718	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8719	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8720	}
8721	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8722	{
8723	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8724
8725	if ( !fIsNestedGuest
8726	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8727	hmR0VmxApicSetTprThreshold(pVmcsInfo, u8Interrupt >> 4);
8728	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8729
8730	/*
8731	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8732	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8733	* need to re-set this force-flag here.
8734	*/
8735	}
8736	else
8737	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8738	}
8739	else if (!fIsNestedGuest)
8740	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8741	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCS controls. */
8742	}
8743
8744	return VINF_SUCCESS;
8745	}
8746
8747
8748	/**
8749	* Injects any pending events into the guest if the guest is in a state to
8750	* receive them.
8751	*
8752	* @returns Strict VBox status code (i.e. informational status codes too).
8753	* @param pVCpu The cross context virtual CPU structure.
8754	* @param pVmxTransient The VMX-transient structure.
8755	* @param fIntrState The VT-x guest-interruptibility state.
8756	* @param fStepping Whether we are single-stepping the guest using the
8757	* hypervisor debugger and should return
8758	* VINF_EM_DBG_STEPPED if the event was dispatched
8759	* directly.
8760	*/
8761	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8762	{
8763	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8764	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8765
8766	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8767	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8768
8769	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8770	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8771	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8772	Assert(!TRPMHasTrap(pVCpu));
8773
8774	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8775	if (pVCpu->hm.s.Event.fPending)
8776	{
8777	/*
8778	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8779	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8780	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8781	*
8782	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8783	*/
8784	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8785	#ifdef VBOX_STRICT
8786	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8787	{
8788	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8789	Assert(!fBlockInt);
8790	Assert(!fBlockSti);
8791	Assert(!fBlockMovSS);
8792	}
8793	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8794	{
8795	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8796	Assert(!fBlockSti);
8797	Assert(!fBlockMovSS);
8798	Assert(!fBlockNmi);
8799	}
8800	#endif
8801	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8802	uIntType));
8803
8804	/*
8805	* Inject the event and get any changes to the guest-interruptibility state.
8806	*
8807	* The guest-interruptibility state may need to be updated if we inject the event
8808	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8809	*/
8810	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8811	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8812
8813	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8814	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8815	else
8816	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8817	}
8818
8819	/*
8820	* Update the guest-interruptibility state.
8821	*
8822	* This is required for the real-on-v86 software interrupt injection case above, as well as
8823	* updates to the guest state from ring-3 or IEM/REM.
8824	*/
8825	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8826	AssertRC(rc);
8827
8828	/*
8829	* There's no need to clear the VM-entry interruption-information field here if we're not
8830	* injecting anything. VT-x clears the valid bit on every VM-exit.
8831	*
8832	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8833	*/
8834
8835	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8836	NOREF(fBlockMovSS); NOREF(fBlockSti);
8837	return rcStrict;
8838	}
8839
8840
8841	/**
8842	* Enters the VT-x session.
8843	*
8844	* @returns VBox status code.
8845	* @param pVCpu The cross context virtual CPU structure.
8846	*/
8847	VMMR0DECL(int) VMXR0Enter(PVMCPUCC pVCpu)
8848	{
8849	AssertPtr(pVCpu);
8850	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8851	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8852
8853	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8854	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8855	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8856
8857	#ifdef VBOX_STRICT
8858	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8859	RTCCUINTREG uHostCr4 = ASMGetCR4();
8860	if (!(uHostCr4 & X86_CR4_VMXE))
8861	{
8862	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8863	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8864	}
8865	#endif
8866
8867	/*
8868	* Load the appropriate VMCS as the current and active one.
8869	*/
8870	PVMXVMCSINFO pVmcsInfo;
8871	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8872	if (!fInNestedGuestMode)
8873	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8874	else
8875	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8876	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8877	if (RT_SUCCESS(rc))
8878	{
8879	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8880	pVCpu->hm.s.fLeaveDone = false;
8881	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8882
8883	/*
8884	* Do the EMT scheduled L1D flush here if needed.
8885	*/
8886	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8887	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8888	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8889	hmR0MdsClear();
8890	}
8891	return rc;
8892	}
8893
8894
8895	/**
8896	* The thread-context callback (only on platforms which support it).
8897	*
8898	* @param enmEvent The thread-context event.
8899	* @param pVCpu The cross context virtual CPU structure.
8900	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8901	* @thread EMT(pVCpu)
8902	*/
8903	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
8904	{
8905	AssertPtr(pVCpu);
8906	RT_NOREF1(fGlobalInit);
8907
8908	switch (enmEvent)
8909	{
8910	case RTTHREADCTXEVENT_OUT:
8911	{
8912	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8913	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8914	VMCPU_ASSERT_EMT(pVCpu);
8915
8916	/* No longjmps (logger flushes, locks) in this fragile context. */
8917	VMMRZCallRing3Disable(pVCpu);
8918	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8919
8920	/* Restore host-state (FPU, debug etc.) */
8921	if (!pVCpu->hm.s.fLeaveDone)
8922	{
8923	/*
8924	* Do -not- import the guest-state here as we might already be in the middle of importing
8925	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8926	*/
8927	hmR0VmxLeave(pVCpu, false /* fImportState */);
8928	pVCpu->hm.s.fLeaveDone = true;
8929	}
8930
8931	/* Leave HM context, takes care of local init (term). */
8932	int rc = HMR0LeaveCpu(pVCpu);
8933	AssertRC(rc);
8934
8935	/* Restore longjmp state. */
8936	VMMRZCallRing3Enable(pVCpu);
8937	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8938	break;
8939	}
8940
8941	case RTTHREADCTXEVENT_IN:
8942	{
8943	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8944	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8945	VMCPU_ASSERT_EMT(pVCpu);
8946
8947	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8948	VMMRZCallRing3Disable(pVCpu);
8949	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8950
8951	/* Initialize the bare minimum state required for HM. This takes care of
8952	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8953	int rc = hmR0EnterCpu(pVCpu);
8954	AssertRC(rc);
8955	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8956	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8957
8958	/* Load the active VMCS as the current one. */
8959	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8960	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8961	AssertRC(rc);
8962	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8963	pVCpu->hm.s.fLeaveDone = false;
8964
8965	/* Do the EMT scheduled L1D flush if needed. */
8966	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8967	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8968
8969	/* Restore longjmp state. */
8970	VMMRZCallRing3Enable(pVCpu);
8971	break;
8972	}
8973
8974	default:
8975	break;
8976	}
8977	}
8978
8979
8980	/**
8981	* Exports the host state into the VMCS host-state area.
8982	* Sets up the VM-exit MSR-load area.
8983	*
8984	* The CPU state will be loaded from these fields on every successful VM-exit.
8985	*
8986	* @returns VBox status code.
8987	* @param pVCpu The cross context virtual CPU structure.
8988	*
8989	* @remarks No-long-jump zone!!!
8990	*/
8991	static int hmR0VmxExportHostState(PVMCPUCC pVCpu)
8992	{
8993	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8994
8995	int rc = VINF_SUCCESS;
8996	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8997	{
8998	hmR0VmxExportHostControlRegs();
8999
9000	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9001	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9002
9003	hmR0VmxExportHostMsrs(pVCpu);
9004
9005	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9006	}
9007	return rc;
9008	}
9009
9010
9011	/**
9012	* Saves the host state in the VMCS host-state.
9013	*
9014	* @returns VBox status code.
9015	* @param pVCpu The cross context virtual CPU structure.
9016	*
9017	* @remarks No-long-jump zone!!!
9018	*/
9019	VMMR0DECL(int) VMXR0ExportHostState(PVMCPUCC pVCpu)
9020	{
9021	AssertPtr(pVCpu);
9022	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9023
9024	/*
9025	* Export the host state here while entering HM context.
9026	* When thread-context hooks are used, we might get preempted and have to re-save the host
9027	* state but most of the time we won't be, so do it here before we disable interrupts.
9028	*/
9029	return hmR0VmxExportHostState(pVCpu);
9030	}
9031
9032
9033	/**
9034	* Exports the guest state into the VMCS guest-state area.
9035	*
9036	* The will typically be done before VM-entry when the guest-CPU state and the
9037	* VMCS state may potentially be out of sync.
9038	*
9039	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9040	* VM-entry controls.
9041	* Sets up the appropriate VMX non-root function to execute guest code based on
9042	* the guest CPU mode.
9043	*
9044	* @returns VBox strict status code.
9045	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9046	* without unrestricted guest execution and the VMMDev is not presently
9047	* mapped (e.g. EFI32).
9048	*
9049	* @param pVCpu The cross context virtual CPU structure.
9050	* @param pVmxTransient The VMX-transient structure.
9051	*
9052	* @remarks No-long-jump zone!!!
9053	*/
9054	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9055	{
9056	AssertPtr(pVCpu);
9057	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9058	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9059
9060	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9061
9062	/*
9063	* Determine real-on-v86 mode.
9064	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9065	*/
9066	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9067	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9068	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9069	pVmcsInfo->RealMode. fRealOnV86Active = false;
9070	else
9071	{
9072	Assert(!pVmxTransient->fIsNestedGuest);
9073	pVmcsInfo->RealMode.fRealOnV86Active = true;
9074	}
9075
9076	/*
9077	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9078	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9079	*/
9080	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9081	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9082	* be a need to evaluate this everytime since I'm pretty sure we intercept
9083	* all guest paging mode changes. */
9084	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9085	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9086
9087	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9088	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9089
9090	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9091	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9092
9093	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9094	if (rcStrict == VINF_SUCCESS)
9095	{ /* likely */ }
9096	else
9097	{
9098	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9099	return rcStrict;
9100	}
9101
9102	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9103	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9104
9105	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9106	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9107
9108	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9109	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9110
9111	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9112	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9113
9114	rc = hmR0VmxExportGuestRip(pVCpu);
9115	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9116	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9117	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9118
9119	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9120	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9121
9122	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9123	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9124	\| HM_CHANGED_GUEST_CR2
9125	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9126	\| HM_CHANGED_GUEST_X87
9127	\| HM_CHANGED_GUEST_SSE_AVX
9128	\| HM_CHANGED_GUEST_OTHER_XSAVE
9129	\| HM_CHANGED_GUEST_XCRx
9130	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9131	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9132	\| HM_CHANGED_GUEST_TSC_AUX
9133	\| HM_CHANGED_GUEST_OTHER_MSRS
9134	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9135
9136	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9137	return rc;
9138	}
9139
9140
9141	/**
9142	* Exports the state shared between the host and guest into the VMCS.
9143	*
9144	* @param pVCpu The cross context virtual CPU structure.
9145	* @param pVmxTransient The VMX-transient structure.
9146	*
9147	* @remarks No-long-jump zone!!!
9148	*/
9149	static void hmR0VmxExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9150	{
9151	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9152	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9153
9154	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9155	{
9156	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9157	AssertRC(rc);
9158	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9159
9160	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9161	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9162	{
9163	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9164	AssertRC(rc);
9165	}
9166	}
9167
9168	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9169	{
9170	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9171	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9172	}
9173
9174	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9175	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9176	}
9177
9178
9179	/**
9180	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9181	*
9182	* @returns Strict VBox status code (i.e. informational status codes too).
9183	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9184	* without unrestricted guest execution and the VMMDev is not presently
9185	* mapped (e.g. EFI32).
9186	*
9187	* @param pVCpu The cross context virtual CPU structure.
9188	* @param pVmxTransient The VMX-transient structure.
9189	*
9190	* @remarks No-long-jump zone!!!
9191	*/
9192	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9193	{
9194	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9195	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9196	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9197
9198	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9199	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9200	#endif
9201
9202	/*
9203	* For many exits it's only RIP that changes and hence try to export it first
9204	* without going through a lot of change flag checks.
9205	*/
9206	VBOXSTRICTRC rcStrict;
9207	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9208	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9209	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9210	{
9211	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9212	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9213	{ /* likely */}
9214	else
9215	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9216	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9217	}
9218	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9219	{
9220	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9221	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9222	{ /* likely */}
9223	else
9224	{
9225	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9226	VBOXSTRICTRC_VAL(rcStrict)));
9227	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9228	return rcStrict;
9229	}
9230	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9231	}
9232	else
9233	rcStrict = VINF_SUCCESS;
9234
9235	#ifdef VBOX_STRICT
9236	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9237	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9238	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9239	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9240	("fCtxChanged=%#RX64\n", fCtxChanged));
9241	#endif
9242	return rcStrict;
9243	}
9244
9245
9246	/**
9247	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9248	* and update error record fields accordingly.
9249	*
9250	* @returns VMX_IGS_* error codes.
9251	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9252	* wrong with the guest state.
9253	*
9254	* @param pVCpu The cross context virtual CPU structure.
9255	* @param pVmcsInfo The VMCS info. object.
9256	*
9257	* @remarks This function assumes our cache of the VMCS controls
9258	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9259	*/
9260	static uint32_t hmR0VmxCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9261	{
9262	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9263	#define HMVMX_CHECK_BREAK(expr, err) do { \
9264	if (!(expr)) { uError = (err); break; } \
9265	} while (0)
9266
9267	int rc;
9268	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9269	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9270	uint32_t uError = VMX_IGS_ERROR;
9271	uint32_t u32Val;
9272	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9273
9274	do
9275	{
9276	/*
9277	* CR0.
9278	*/
9279	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9280	uint64_t fSetCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9281	uint64_t const fZapCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9282	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
9283	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9284	if (fUnrestrictedGuest)
9285	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
9286
9287	uint64_t u64GuestCr0;
9288	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64GuestCr0);
9289	AssertRC(rc);
9290	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9291	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9292	if ( !fUnrestrictedGuest
9293	&& (u64GuestCr0 & X86_CR0_PG)
9294	&& !(u64GuestCr0 & X86_CR0_PE))
9295	{
9296	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9297	}
9298
9299	/*
9300	* CR4.
9301	*/
9302	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9303	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9304	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9305
9306	uint64_t u64GuestCr4;
9307	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64GuestCr4);
9308	AssertRC(rc);
9309	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9310	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9311
9312	/*
9313	* IA32_DEBUGCTL MSR.
9314	*/
9315	uint64_t u64Val;
9316	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9317	AssertRC(rc);
9318	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9319	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9320	{
9321	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9322	}
9323	uint64_t u64DebugCtlMsr = u64Val;
9324
9325	#ifdef VBOX_STRICT
9326	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9327	AssertRC(rc);
9328	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9329	#endif
9330	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9331
9332	/*
9333	* RIP and RFLAGS.
9334	*/
9335	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
9336	AssertRC(rc);
9337	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9338	if ( !fLongModeGuest
9339	\|\| !pCtx->cs.Attr.n.u1Long)
9340	{
9341	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9342	}
9343	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9344	* must be identical if the "IA-32e mode guest" VM-entry
9345	* control is 1 and CS.L is 1. No check applies if the
9346	* CPU supports 64 linear-address bits. */
9347
9348	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9349	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9350	AssertRC(rc);
9351	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9352	VMX_IGS_RFLAGS_RESERVED);
9353	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9354	uint32_t const u32Eflags = u64Val;
9355
9356	if ( fLongModeGuest
9357	\|\| ( fUnrestrictedGuest
9358	&& !(u64GuestCr0 & X86_CR0_PE)))
9359	{
9360	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9361	}
9362
9363	uint32_t u32EntryInfo;
9364	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9365	AssertRC(rc);
9366	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9367	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9368
9369	/*
9370	* 64-bit checks.
9371	*/
9372	if (fLongModeGuest)
9373	{
9374	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9375	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9376	}
9377
9378	if ( !fLongModeGuest
9379	&& (u64GuestCr4 & X86_CR4_PCIDE))
9380	{
9381	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9382	}
9383
9384	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9385	* 51:32 beyond the processor's physical-address width are 0. */
9386
9387	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9388	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9389	{
9390	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9391	}
9392
9393	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9394	AssertRC(rc);
9395	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9396
9397	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9398	AssertRC(rc);
9399	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9400
9401	/*
9402	* PERF_GLOBAL MSR.
9403	*/
9404	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9405	{
9406	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9407	AssertRC(rc);
9408	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9409	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9410	}
9411
9412	/*
9413	* PAT MSR.
9414	*/
9415	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9416	{
9417	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9418	AssertRC(rc);
9419	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9420	for (unsigned i = 0; i < 8; i++)
9421	{
9422	uint8_t u8Val = (u64Val & 0xff);
9423	if ( u8Val != 0 /* UC */
9424	&& u8Val != 1 /* WC */
9425	&& u8Val != 4 /* WT */
9426	&& u8Val != 5 /* WP */
9427	&& u8Val != 6 /* WB */
9428	&& u8Val != 7 /* UC- */)
9429	{
9430	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9431	}
9432	u64Val >>= 8;
9433	}
9434	}
9435
9436	/*
9437	* EFER MSR.
9438	*/
9439	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9440	{
9441	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9442	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9443	AssertRC(rc);
9444	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9445	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9446	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9447	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9448	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9449	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9450	* iemVmxVmentryCheckGuestState(). */
9451	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9452	\|\| !(u64GuestCr0 & X86_CR0_PG)
9453	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9454	VMX_IGS_EFER_LMA_LME_MISMATCH);
9455	}
9456
9457	/*
9458	* Segment registers.
9459	*/
9460	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9461	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9462	if (!(u32Eflags & X86_EFL_VM))
9463	{
9464	/* CS */
9465	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9466	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9467	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9468	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9469	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9470	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9471	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9472	/* CS cannot be loaded with NULL in protected mode. */
9473	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9474	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9475	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9476	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9477	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9478	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9479	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9480	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9481	else
9482	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9483
9484	/* SS */
9485	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9486	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9487	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9488	if ( !(pCtx->cr0 & X86_CR0_PE)
9489	\|\| pCtx->cs.Attr.n.u4Type == 3)
9490	{
9491	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9492	}
9493	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9494	{
9495	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9496	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9497	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9498	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9499	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9500	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9501	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9502	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9503	}
9504
9505	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9506	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9507	{
9508	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9509	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9510	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9511	\|\| pCtx->ds.Attr.n.u4Type > 11
9512	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9513	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9514	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9515	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9516	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9517	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9518	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9519	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9520	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9521	}
9522	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9523	{
9524	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9525	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9526	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9527	\|\| pCtx->es.Attr.n.u4Type > 11
9528	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9529	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9530	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9531	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9532	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9533	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9534	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9535	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9536	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9537	}
9538	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9539	{
9540	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9541	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9542	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9543	\|\| pCtx->fs.Attr.n.u4Type > 11
9544	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9545	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9546	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9547	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9548	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9549	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9550	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9551	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9552	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9553	}
9554	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9555	{
9556	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9557	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9558	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9559	\|\| pCtx->gs.Attr.n.u4Type > 11
9560	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9561	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9562	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9563	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9564	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9565	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9566	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9567	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9568	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9569	}
9570	/* 64-bit capable CPUs. */
9571	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9572	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9573	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9574	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9575	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9576	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9577	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9578	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9579	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9580	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9581	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9582	}
9583	else
9584	{
9585	/* V86 mode checks. */
9586	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9587	if (pVmcsInfo->RealMode.fRealOnV86Active)
9588	{
9589	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9590	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9591	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9592	}
9593	else
9594	{
9595	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9596	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9597	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9598	}
9599
9600	/* CS */
9601	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9602	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9603	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9604	/* SS */
9605	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9606	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9607	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9608	/* DS */
9609	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9610	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9611	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9612	/* ES */
9613	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9614	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9615	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9616	/* FS */
9617	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9618	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9619	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9620	/* GS */
9621	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9622	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9623	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9624	/* 64-bit capable CPUs. */
9625	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9626	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9627	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9628	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9629	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9630	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9631	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9632	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9633	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9634	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9635	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9636	}
9637
9638	/*
9639	* TR.
9640	*/
9641	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9642	/* 64-bit capable CPUs. */
9643	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9644	if (fLongModeGuest)
9645	{
9646	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9647	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9648	}
9649	else
9650	{
9651	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9652	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9653	VMX_IGS_TR_ATTR_TYPE_INVALID);
9654	}
9655	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9656	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9657	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9658	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9659	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9660	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9661	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9662	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9663
9664	/*
9665	* GDTR and IDTR (64-bit capable checks).
9666	*/
9667	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9668	AssertRC(rc);
9669	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9670
9671	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9672	AssertRC(rc);
9673	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9674
9675	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9676	AssertRC(rc);
9677	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9678
9679	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9680	AssertRC(rc);
9681	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9682
9683	/*
9684	* Guest Non-Register State.
9685	*/
9686	/* Activity State. */
9687	uint32_t u32ActivityState;
9688	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9689	AssertRC(rc);
9690	HMVMX_CHECK_BREAK( !u32ActivityState
9691	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9692	VMX_IGS_ACTIVITY_STATE_INVALID);
9693	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9694	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9695	uint32_t u32IntrState;
9696	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9697	AssertRC(rc);
9698	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9699	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9700	{
9701	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9702	}
9703
9704	/** @todo Activity state and injecting interrupts. Left as a todo since we
9705	* currently don't use activity states but ACTIVE. */
9706
9707	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9708	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9709
9710	/* Guest interruptibility-state. */
9711	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9712	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9713	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9714	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9715	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9716	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9717	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9718	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9719	{
9720	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9721	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9722	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9723	}
9724	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9725	{
9726	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9727	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9728	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9729	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9730	}
9731	/** @todo Assumes the processor is not in SMM. */
9732	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9733	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9734	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9735	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9736	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9737	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9738	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9739	{
9740	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9741	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9742	}
9743
9744	/* Pending debug exceptions. */
9745	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9746	AssertRC(rc);
9747	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9748	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9749	u32Val = u64Val; /* For pending debug exceptions checks below. */
9750
9751	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9752	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9753	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9754	{
9755	if ( (u32Eflags & X86_EFL_TF)
9756	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9757	{
9758	/* Bit 14 is PendingDebug.BS. */
9759	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9760	}
9761	if ( !(u32Eflags & X86_EFL_TF)
9762	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9763	{
9764	/* Bit 14 is PendingDebug.BS. */
9765	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9766	}
9767	}
9768
9769	/* VMCS link pointer. */
9770	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9771	AssertRC(rc);
9772	if (u64Val != UINT64_C(0xffffffffffffffff))
9773	{
9774	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9775	/** @todo Bits beyond the processor's physical-address width MBZ. */
9776	/** @todo SMM checks. */
9777	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
9778	Assert(pVmcsInfo->pvShadowVmcs);
9779	VMXVMCSREVID VmcsRevId;
9780	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
9781	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
9782	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
9783	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
9784	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
9785	}
9786
9787	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9788	* not using nested paging? */
9789	if ( pVM->hm.s.fNestedPaging
9790	&& !fLongModeGuest
9791	&& CPUMIsGuestInPAEModeEx(pCtx))
9792	{
9793	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9794	AssertRC(rc);
9795	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9796
9797	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9798	AssertRC(rc);
9799	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9800
9801	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9802	AssertRC(rc);
9803	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9804
9805	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9806	AssertRC(rc);
9807	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9808	}
9809
9810	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9811	if (uError == VMX_IGS_ERROR)
9812	uError = VMX_IGS_REASON_NOT_FOUND;
9813	} while (0);
9814
9815	pVCpu->hm.s.u32HMError = uError;
9816	return uError;
9817
9818	#undef HMVMX_ERROR_BREAK
9819	#undef HMVMX_CHECK_BREAK
9820	}
9821
9822
9823	/**
9824	* Map the APIC-access page for virtualizing APIC accesses.
9825	*
9826	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
9827	* this not done as part of exporting guest state, see @bugref{8721}.
9828	*
9829	* @returns VBox status code.
9830	* @param pVCpu The cross context virtual CPU structure.
9831	*/
9832	static int hmR0VmxMapHCApicAccessPage(PVMCPUCC pVCpu)
9833	{
9834	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9835	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9836
9837	Assert(PDMHasApic(pVM));
9838	Assert(u64MsrApicBase);
9839
9840	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9841	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9842
9843	/* Unalias the existing mapping. */
9844	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9845	AssertRCReturn(rc, rc);
9846
9847	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9848	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9849	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9850	AssertRCReturn(rc, rc);
9851
9852	/* Update the per-VCPU cache of the APIC base MSR. */
9853	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9854	return VINF_SUCCESS;
9855	}
9856
9857
9858	/**
9859	* Worker function passed to RTMpOnSpecific() that is to be called on the target
9860	* CPU.
9861	*
9862	* @param idCpu The ID for the CPU the function is called on.
9863	* @param pvUser1 Null, not used.
9864	* @param pvUser2 Null, not used.
9865	*/
9866	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
9867	{
9868	RT_NOREF3(idCpu, pvUser1, pvUser2);
9869	VMXDispatchHostNmi();
9870	}
9871
9872
9873	/**
9874	* Dispatching an NMI on the host CPU that received it.
9875	*
9876	* @returns VBox status code.
9877	* @param pVCpu The cross context virtual CPU structure.
9878	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
9879	* executing when receiving the host NMI in VMX non-root
9880	* operation.
9881	*/
9882	static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9883	{
9884	RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
9885	Assert(idCpu != NIL_RTCPUID);
9886
9887	/*
9888	* We don't want to delay dispatching the NMI any more than we have to. However,
9889	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
9890	* after executing guest or nested-guest code for the following reasons:
9891	*
9892	* - We would need to perform VMREADs with interrupts disabled and is orders of
9893	* magnitude worse when we run as a nested hypervisor without VMCS shadowing
9894	* supported by the host hypervisor.
9895	*
9896	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
9897	* longer period of time just for handling an edge case like host NMIs which do
9898	* not occur nearly as frequently as other VM-exits.
9899	*
9900	* Let's cover the most likely scenario first. Check if we are on the target CPU
9901	* and dispatch the NMI right away. This should be much faster than calling into
9902	* RTMpOnSpecific() machinery.
9903	*/
9904	bool fDispatched = false;
9905	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
9906	if (idCpu == RTMpCpuId())
9907	{
9908	VMXDispatchHostNmi();
9909	fDispatched = true;
9910	}
9911	ASMSetFlags(fEFlags);
9912	if (fDispatched)
9913	{
9914	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9915	return VINF_SUCCESS;
9916	}
9917
9918	/*
9919	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
9920	* there should be no race or recursion even if we are unlucky enough to be preempted
9921	* (to the target CPU) without dispatching the host NMI above.
9922	*/
9923	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
9924	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
9925	}
9926
9927
9928	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9929	/**
9930	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9931	* nested-guest using hardware-assisted VMX.
9932	*
9933	* @param pVCpu The cross context virtual CPU structure.
9934	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9935	* @param pVmcsInfoGst The guest VMCS info. object.
9936	*/
9937	static void hmR0VmxMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9938	{
9939	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
9940	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9941	Assert(pu64MsrBitmap);
9942
9943	/*
9944	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9945	* MSR that is intercepted by the guest is also intercepted while executing the
9946	* nested-guest using hardware-assisted VMX.
9947	*
9948	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
9949	* nested-guest VM-exit even if the outer guest is not intercepting some
9950	* MSRs. We cannot assume the caller has initialized the nested-guest
9951	* MSR bitmap in this case.
9952	*
9953	* The nested hypervisor may also switch whether it uses MSR bitmaps for
9954	* each VM-entry, hence initializing it once per-VM while setting up the
9955	* nested-guest VMCS is not sufficient.
9956	*/
9957	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9958	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
9959	{
9960	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9961	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9962	Assert(pu64MsrBitmapNstGst);
9963	Assert(pu64MsrBitmapGst);
9964
9965	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
9966	for (uint32_t i = 0; i < cFrags; i++)
9967	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9968	}
9969	else
9970	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
9971	}
9972
9973
9974	/**
9975	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9976	* hardware-assisted VMX execution of the nested-guest.
9977	*
9978	* For a guest, we don't modify these controls once we set up the VMCS and hence
9979	* this function is never called.
9980	*
9981	* For nested-guests since the nested hypervisor provides these controls on every
9982	* nested-guest VM-entry and could potentially change them everytime we need to
9983	* merge them before every nested-guest VM-entry.
9984	*
9985	* @returns VBox status code.
9986	* @param pVCpu The cross context virtual CPU structure.
9987	*/
9988	static int hmR0VmxMergeVmcsNested(PVMCPUCC pVCpu)
9989	{
9990	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9991	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9992	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9993	Assert(pVmcsNstGst);
9994
9995	/*
9996	* Merge the controls with the requirements of the guest VMCS.
9997	*
9998	* We do not need to validate the nested-guest VMX features specified in the nested-guest
9999	* VMCS with the features supported by the physical CPU as it's already done by the
10000	* VMLAUNCH/VMRESUME instruction emulation.
10001	*
10002	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10003	* derived from the VMX features supported by the physical CPU.
10004	*/
10005
10006	/* Pin-based VM-execution controls. */
10007	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10008
10009	/* Processor-based VM-execution controls. */
10010	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10011	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10012	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10013	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10014	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10015
10016	/* Secondary processor-based VM-execution controls. */
10017	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10018	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10019	\| VMX_PROC_CTLS2_INVPCID
10020	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10021	\| VMX_PROC_CTLS2_RDTSCP
10022	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10023	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10024	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10025	\| VMX_PROC_CTLS2_VMFUNC));
10026
10027	/*
10028	* VM-entry controls:
10029	* These controls contains state that depends on the nested-guest state (primarily
10030	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10031	* VM-exit. Although the nested hypervisor cannot change it, we need to in order to
10032	* properly continue executing the nested-guest if the EFER MSR changes but does not
10033	* cause a nested-guest VM-exits.
10034	*
10035	* VM-exit controls:
10036	* These controls specify the host state on return. We cannot use the controls from
10037	* the nested hypervisor state as is as it would contain the guest state rather than
10038	* the host state. Since the host state is subject to change (e.g. preemption, trips
10039	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10040	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10041	*
10042	* VM-entry MSR-load:
10043	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10044	* context by the VMLAUNCH/VMRESUME instruction emulation.
10045	*
10046	* VM-exit MSR-store:
10047	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10048	* back into the VM-exit MSR-store area.
10049	*
10050	* VM-exit MSR-load areas:
10051	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10052	* can entirely ignore what the nested hypervisor wants to load here.
10053	*/
10054
10055	/*
10056	* Exception bitmap.
10057	*
10058	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10059	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10060	* code more flexible if intercepting exceptions become more dynamic in the future we do
10061	* it as part of exporting the nested-guest state.
10062	*/
10063	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10064
10065	/*
10066	* CR0/CR4 guest/host mask.
10067	*
10068	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10069	* cause VM-exits, so we need to merge them here.
10070	*/
10071	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10072	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10073
10074	/*
10075	* Page-fault error-code mask and match.
10076	*
10077	* Although we require unrestricted guest execution (and thereby nested-paging) for
10078	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10079	* normally intercept #PFs, it might intercept them for debugging purposes.
10080	*
10081	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10082	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10083	*/
10084	uint32_t u32XcptPFMask;
10085	uint32_t u32XcptPFMatch;
10086	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10087	{
10088	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10089	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10090	}
10091	else
10092	{
10093	u32XcptPFMask = 0;
10094	u32XcptPFMatch = 0;
10095	}
10096
10097	/*
10098	* Pause-Loop exiting.
10099	*/
10100	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10101	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10102
10103	/*
10104	* Pending debug exceptions.
10105	* Currently just copy whatever the nested-guest provides us.
10106	*/
10107	uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
10108
10109	/*
10110	* I/O Bitmap.
10111	*
10112	* We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
10113	* intercept all I/O port accesses.
10114	*/
10115	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10116	Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
10117
10118	/*
10119	* VMCS shadowing.
10120	*
10121	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10122	* enabled while executing the nested-guest.
10123	*/
10124	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10125
10126	/*
10127	* APIC-access page.
10128	*/
10129	RTHCPHYS HCPhysApicAccess;
10130	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10131	{
10132	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10133	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10134
10135	/** @todo NSTVMX: This is not really correct but currently is required to make
10136	* things work. We need to re-enable the page handler when we fallback to
10137	* IEM execution of the nested-guest! */
10138	PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
10139
10140	void *pvPage;
10141	PGMPAGEMAPLOCK PgLockApicAccess;
10142	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
10143	if (RT_SUCCESS(rc))
10144	{
10145	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10146	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10147
10148	/** @todo Handle proper releasing of page-mapping lock later. */
10149	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
10150	}
10151	else
10152	return rc;
10153	}
10154	else
10155	HCPhysApicAccess = 0;
10156
10157	/*
10158	* Virtual-APIC page and TPR threshold.
10159	*/
10160	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10161	RTHCPHYS HCPhysVirtApic;
10162	uint32_t u32TprThreshold;
10163	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10164	{
10165	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10166	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10167
10168	void *pvPage;
10169	PGMPAGEMAPLOCK PgLockVirtApic;
10170	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
10171	if (RT_SUCCESS(rc))
10172	{
10173	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10174	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10175
10176	/** @todo Handle proper releasing of page-mapping lock later. */
10177	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
10178	}
10179	else
10180	return rc;
10181
10182	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10183	}
10184	else
10185	{
10186	HCPhysVirtApic = 0;
10187	u32TprThreshold = 0;
10188
10189	/*
10190	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10191	* used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
10192	* be taken care of by EPT/shadow paging.
10193	*/
10194	if (pVM->hm.s.fAllow64BitGuests)
10195	{
10196	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10197	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10198	}
10199	}
10200
10201	/*
10202	* Validate basic assumptions.
10203	*/
10204	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10205	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10206	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10207
10208	/*
10209	* Commit it to the nested-guest VMCS.
10210	*/
10211	int rc = VINF_SUCCESS;
10212	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10213	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10214	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10215	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10216	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10217	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10218	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10219	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10220	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10221	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10222	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10223	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10224	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10225	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10226	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10227	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10228	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10229	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10230	{
10231	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10232	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10233	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10234	}
10235	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10236	{
10237	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10238	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10239	}
10240	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10241	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10242	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
10243	AssertRC(rc);
10244
10245	/*
10246	* Update the nested-guest VMCS cache.
10247	*/
10248	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10249	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10250	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10251	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10252	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10253	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10254	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10255	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10256	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10257
10258	/*
10259	* We need to flush the TLB if we are switching the APIC-access page address.
10260	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10261	*/
10262	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10263	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10264
10265	/*
10266	* MSR bitmap.
10267	*
10268	* The MSR bitmap address has already been initialized while setting up the nested-guest
10269	* VMCS, here we need to merge the MSR bitmaps.
10270	*/
10271	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10272	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10273
10274	return VINF_SUCCESS;
10275	}
10276	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10277
10278
10279	/**
10280	* Does the preparations before executing guest code in VT-x.
10281	*
10282	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10283	* recompiler/IEM. We must be cautious what we do here regarding committing
10284	* guest-state information into the VMCS assuming we assuredly execute the
10285	* guest in VT-x mode.
10286	*
10287	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10288	* the common-state (TRPM/forceflags), we must undo those changes so that the
10289	* recompiler/IEM can (and should) use them when it resumes guest execution.
10290	* Otherwise such operations must be done when we can no longer exit to ring-3.
10291	*
10292	* @returns Strict VBox status code (i.e. informational status codes too).
10293	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10294	* have been disabled.
10295	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10296	* pending events).
10297	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10298	* double-fault into the guest.
10299	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10300	* dispatched directly.
10301	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10302	*
10303	* @param pVCpu The cross context virtual CPU structure.
10304	* @param pVmxTransient The VMX-transient structure.
10305	* @param fStepping Whether we are single-stepping the guest in the
10306	* hypervisor debugger. Makes us ignore some of the reasons
10307	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10308	* if event dispatching took place.
10309	*/
10310	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10311	{
10312	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10313
10314	Log4Func(("fIsNested=%RTbool fStepping=%RTbool\n", pVmxTransient->fIsNestedGuest, fStepping));
10315
10316	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10317	if (pVmxTransient->fIsNestedGuest)
10318	{
10319	RT_NOREF2(pVCpu, fStepping);
10320	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10321	return VINF_EM_RESCHEDULE_REM;
10322	}
10323	#endif
10324
10325	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10326	PGMRZDynMapFlushAutoSet(pVCpu);
10327	#endif
10328
10329	/*
10330	* Check and process force flag actions, some of which might require us to go back to ring-3.
10331	*/
10332	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10333	if (rcStrict == VINF_SUCCESS)
10334	{
10335	/* FFs don't get set all the time. */
10336	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10337	if ( pVmxTransient->fIsNestedGuest
10338	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10339	{
10340	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10341	return VINF_VMX_VMEXIT;
10342	}
10343	#endif
10344	}
10345	else
10346	return rcStrict;
10347
10348	/*
10349	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10350	*/
10351	/** @todo Doing this from ring-3 after VM setup phase causes a
10352	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
10353	* idea why atm. */
10354	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10355	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10356	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10357	&& PDMHasApic(pVM))
10358	{
10359	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10360	AssertRCReturn(rc, rc);
10361	}
10362
10363	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10364	/*
10365	* Merge guest VMCS controls with the nested-guest VMCS controls.
10366	*
10367	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10368	* saved state), we should be okay with merging controls as we initialize the
10369	* guest VMCS controls as part of VM setup phase.
10370	*/
10371	if ( pVmxTransient->fIsNestedGuest
10372	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10373	{
10374	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10375	AssertRCReturn(rc, rc);
10376	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10377	}
10378	#endif
10379
10380	/*
10381	* Evaluate events to be injected into the guest.
10382	*
10383	* Events in TRPM can be injected without inspecting the guest state.
10384	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10385	* guest to cause a VM-exit the next time they are ready to receive the event.
10386	*
10387	* With nested-guests, evaluating pending events may cause VM-exits. Also, verify
10388	* that the event in TRPM that we will inject using hardware-assisted VMX is -not-
10389	* subject to interecption. Otherwise, we should have checked and injected them
10390	* manually elsewhere (IEM).
10391	*/
10392	if (TRPMHasTrap(pVCpu))
10393	{
10394	Assert(!pVmxTransient->fIsNestedGuest \|\| !CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
10395	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10396	}
10397
10398	uint32_t fIntrState;
10399	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10400
10401	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10402	/*
10403	* While evaluating pending events if something failed (unlikely) or if we were
10404	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10405	*/
10406	if (rcStrict != VINF_SUCCESS)
10407	return rcStrict;
10408	if ( pVmxTransient->fIsNestedGuest
10409	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10410	{
10411	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10412	return VINF_VMX_VMEXIT;
10413	}
10414	#else
10415	Assert(rcStrict == VINF_SUCCESS);
10416	#endif
10417
10418	/*
10419	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10420	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10421	* also result in triple-faulting the VM.
10422	*
10423	* With nested-guests, the above does not apply since unrestricted guest execution is a
10424	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10425	*/
10426	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10427	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10428	{ /* likely */ }
10429	else
10430	{
10431	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10432	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10433	return rcStrict;
10434	}
10435
10436	/*
10437	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10438	* import CR3 themselves. We will need to update them here, as even as late as the above
10439	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10440	* the below force flags to be set.
10441	*/
10442	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10443	{
10444	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10445	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10446	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10447	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10448	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10449	}
10450	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10451	{
10452	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10453	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10454	}
10455
10456	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10457	/* Paranoia. */
10458	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10459	#endif
10460
10461	/*
10462	* No longjmps to ring-3 from this point on!!!
10463	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10464	* This also disables flushing of the R0-logger instance (if any).
10465	*/
10466	VMMRZCallRing3Disable(pVCpu);
10467
10468	/*
10469	* Export the guest state bits.
10470	*
10471	* We cannot perform longjmps while loading the guest state because we do not preserve the
10472	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10473	* CPU migration.
10474	*
10475	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10476	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10477	*/
10478	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10479	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10480	{ /* likely */ }
10481	else
10482	{
10483	VMMRZCallRing3Enable(pVCpu);
10484	return rcStrict;
10485	}
10486
10487	/*
10488	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10489	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10490	* preemption disabled for a while. Since this is purely to aid the
10491	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10492	* disable interrupt on NT.
10493	*
10494	* We need to check for force-flags that could've possible been altered since we last
10495	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10496	* see @bugref{6398}).
10497	*
10498	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10499	* to ring-3 before executing guest code.
10500	*/
10501	pVmxTransient->fEFlags = ASMIntDisableFlags();
10502
10503	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10504	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10505	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10506	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10507	{
10508	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10509	{
10510	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10511	/*
10512	* If we are executing a nested-guest make sure that we should intercept subsequent
10513	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10514	* the VM-exit instruction emulation happy.
10515	*/
10516	if (pVmxTransient->fIsNestedGuest)
10517	CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
10518	#endif
10519
10520	/*
10521	* We've injected any pending events. This is really the point of no return (to ring-3).
10522	*
10523	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10524	* returns from this function, so do -not- enable them here.
10525	*/
10526	pVCpu->hm.s.Event.fPending = false;
10527	return VINF_SUCCESS;
10528	}
10529
10530	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10531	rcStrict = VINF_EM_RAW_INTERRUPT;
10532	}
10533	else
10534	{
10535	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10536	rcStrict = VINF_EM_RAW_TO_R3;
10537	}
10538
10539	ASMSetFlags(pVmxTransient->fEFlags);
10540	VMMRZCallRing3Enable(pVCpu);
10541
10542	return rcStrict;
10543	}
10544
10545
10546	/**
10547	* Final preparations before executing guest code using hardware-assisted VMX.
10548	*
10549	* We can no longer get preempted to a different host CPU and there are no returns
10550	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10551	* failures), this function is not intended to fail sans unrecoverable hardware
10552	* errors.
10553	*
10554	* @param pVCpu The cross context virtual CPU structure.
10555	* @param pVmxTransient The VMX-transient structure.
10556	*
10557	* @remarks Called with preemption disabled.
10558	* @remarks No-long-jump zone!!!
10559	*/
10560	static void hmR0VmxPreRunGuestCommitted(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10561	{
10562	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10563	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10564	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10565	Assert(!pVCpu->hm.s.Event.fPending);
10566
10567	/*
10568	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10569	*/
10570	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10571	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10572
10573	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10574	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10575	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10576	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10577
10578	if (!CPUMIsGuestFPUStateActive(pVCpu))
10579	{
10580	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10581	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10582	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10583	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10584	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10585	}
10586
10587	/*
10588	* Re-export the host state bits as we may've been preempted (only happens when
10589	* thread-context hooks are used or when the VM start function changes) or if
10590	* the host CR0 is modified while loading the guest FPU state above.
10591	*
10592	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10593	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10594	* see @bugref{8432}.
10595	*
10596	* This may also happen when switching to/from a nested-guest VMCS without leaving
10597	* ring-0.
10598	*/
10599	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10600	{
10601	hmR0VmxExportHostState(pVCpu);
10602	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10603	}
10604	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10605
10606	/*
10607	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10608	*/
10609	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10610	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10611	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10612
10613	/*
10614	* Store status of the shared guest/host debug state at the time of VM-entry.
10615	*/
10616	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10617	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10618
10619	/*
10620	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10621	* more than one conditional check. The post-run side of our code shall determine
10622	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10623	*/
10624	if (pVmcsInfo->pbVirtApic)
10625	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10626
10627	/*
10628	* Update the host MSRs values in the VM-exit MSR-load area.
10629	*/
10630	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10631	{
10632	if (pVmcsInfo->cExitMsrLoad > 0)
10633	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10634	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10635	}
10636
10637	/*
10638	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10639	* VMX-preemption timer based on the next virtual sync clock deadline.
10640	*/
10641	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10642	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10643	{
10644	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10645	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10646	}
10647
10648	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10649	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10650	if (!fIsRdtscIntercepted)
10651	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10652	else
10653	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10654
10655	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10656	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10657	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10658	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Record the error reporting info. with the current host CPU. */
10659	pVmcsInfo->idHostCpuState = idCurrentCpu; /* Record the CPU for which the host-state has been exported. */
10660	pVmcsInfo->idHostCpuExec = idCurrentCpu; /* Record the CPU on which we shall execute. */
10661
10662	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10663
10664	TMNotifyStartOfExecution(pVM, pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10665	as we're about to start executing the guest. */
10666
10667	/*
10668	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10669	*
10670	* This is done this late as updating the TSC offsetting/preemption timer above
10671	* figures out if we can skip intercepting RDTSCP by calculating the number of
10672	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10673	*/
10674	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10675	&& !fIsRdtscIntercepted)
10676	{
10677	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10678
10679	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10680	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10681	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10682	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10683	AssertRC(rc);
10684	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10685	pVmxTransient->fRemoveTscAuxMsr = true;
10686	}
10687
10688	#ifdef VBOX_STRICT
10689	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10690	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10691	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10692	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10693	#endif
10694
10695	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10696	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10697	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10698	* see @bugref{9180#c54}. */
10699	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10700	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10701	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10702	#endif
10703	}
10704
10705
10706	/**
10707	* First C routine invoked after running guest code using hardware-assisted VMX.
10708	*
10709	* @param pVCpu The cross context virtual CPU structure.
10710	* @param pVmxTransient The VMX-transient structure.
10711	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10712	*
10713	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10714	*
10715	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10716	* unconditionally when it is safe to do so.
10717	*/
10718	static void hmR0VmxPostRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10719	{
10720	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10721
10722	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10723	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10724	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10725	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10726	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10727	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10728
10729	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10730	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10731	{
10732	uint64_t uGstTsc;
10733	if (!pVmxTransient->fIsNestedGuest)
10734	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10735	else
10736	{
10737	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10738	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10739	}
10740	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10741	}
10742
10743	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10744	TMNotifyEndOfExecution(pVCpu->CTX_SUFF(pVM), pVCpu); /* Notify TM that the guest is no longer running. */
10745	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10746
10747	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10748	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10749	#ifdef VBOX_STRICT
10750	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10751	#endif
10752	Assert(!ASMIntAreEnabled());
10753	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10754	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10755
10756	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
10757	/*
10758	* Clean all the VMCS fields in the transient structure before reading
10759	* anything from the VMCS.
10760	*/
10761	pVmxTransient->uExitReason = 0;
10762	pVmxTransient->uExitIntErrorCode = 0;
10763	pVmxTransient->uExitQual = 0;
10764	pVmxTransient->uGuestLinearAddr = 0;
10765	pVmxTransient->uExitIntInfo = 0;
10766	pVmxTransient->cbExitInstr = 0;
10767	pVmxTransient->ExitInstrInfo.u = 0;
10768	pVmxTransient->uEntryIntInfo = 0;
10769	pVmxTransient->uEntryXcptErrorCode = 0;
10770	pVmxTransient->cbEntryInstr = 0;
10771	pVmxTransient->uIdtVectoringInfo = 0;
10772	pVmxTransient->uIdtVectoringErrorCode = 0;
10773	#endif
10774
10775	/*
10776	* Save the basic VM-exit reason and check if the VM-entry failed.
10777	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10778	*/
10779	uint32_t uExitReason;
10780	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10781	AssertRC(rc);
10782	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10783	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10784
10785	/*
10786	* Log the VM-exit before logging anything else as otherwise it might be a
10787	* tad confusing what happens before and after the world-switch.
10788	*/
10789	HMVMX_LOG_EXIT(pVCpu, uExitReason);
10790
10791	/*
10792	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
10793	* bitmap permissions, if it was added before VM-entry.
10794	*/
10795	if (pVmxTransient->fRemoveTscAuxMsr)
10796	{
10797	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10798	pVmxTransient->fRemoveTscAuxMsr = false;
10799	}
10800
10801	/*
10802	* Check if VMLAUNCH/VMRESUME succeeded.
10803	* If this failed, we cause a guru meditation and cease further execution.
10804	*
10805	* However, if we are executing a nested-guest we might fail if we use the
10806	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10807	*/
10808	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10809	{
10810	/*
10811	* Update the VM-exit history array here even if the VM-entry failed due to:
10812	* - Invalid guest state.
10813	* - MSR loading.
10814	* - Machine-check event.
10815	*
10816	* In any of the above cases we will still have a "valid" VM-exit reason
10817	* despite @a fVMEntryFailed being false.
10818	*
10819	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10820	*
10821	* Note! We don't have CS or RIP at this point. Will probably address that later
10822	* by amending the history entry added here.
10823	*/
10824	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10825	UINT64_MAX, uHostTsc);
10826
10827	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10828	{
10829	VMMRZCallRing3Enable(pVCpu);
10830
10831	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10832	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10833
10834	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
10835	hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
10836	#endif
10837	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10838	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10839	AssertRC(rc);
10840	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10841	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10842	AssertRC(rc);
10843	#else
10844	/*
10845	* Import the guest-interruptibility state always as we need it while evaluating
10846	* injecting events on re-entry.
10847	*
10848	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10849	* checking for real-mode while exporting the state because all bits that cause
10850	* mode changes wrt CR0 are intercepted.
10851	*/
10852	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10853	AssertRC(rc);
10854	#endif
10855
10856	/*
10857	* Sync the TPR shadow with our APIC state.
10858	*/
10859	if ( !pVmxTransient->fIsNestedGuest
10860	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10861	{
10862	Assert(pVmcsInfo->pbVirtApic);
10863	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10864	{
10865	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10866	AssertRC(rc);
10867	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10868	}
10869	}
10870
10871	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10872	return;
10873	}
10874	}
10875	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10876	else if (pVmxTransient->fIsNestedGuest)
10877	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10878	#endif
10879	else
10880	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10881
10882	VMMRZCallRing3Enable(pVCpu);
10883	}
10884
10885
10886	/**
10887	* Runs the guest code using hardware-assisted VMX the normal way.
10888	*
10889	* @returns VBox status code.
10890	* @param pVCpu The cross context virtual CPU structure.
10891	* @param pcLoops Pointer to the number of executed loops.
10892	*/
10893	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
10894	{
10895	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10896	Assert(pcLoops);
10897	Assert(*pcLoops <= cMaxResumeLoops);
10898	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10899
10900	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10901	/*
10902	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
10903	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
10904	* guest VMCS while entering the VMX ring-0 session.
10905	*/
10906	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10907	{
10908	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
10909	if (RT_SUCCESS(rc))
10910	{ /* likely */ }
10911	else
10912	{
10913	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
10914	return rc;
10915	}
10916	}
10917	#endif
10918
10919	VMXTRANSIENT VmxTransient;
10920	RT_ZERO(VmxTransient);
10921	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10922
10923	/* Paranoia. */
10924	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10925
10926	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10927	for (;;)
10928	{
10929	Assert(!HMR0SuspendPending());
10930	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10931	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10932
10933	/*
10934	* Preparatory work for running nested-guest code, this may force us to
10935	* return to ring-3.
10936	*
10937	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10938	*/
10939	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10940	if (rcStrict != VINF_SUCCESS)
10941	break;
10942
10943	/* Interrupts are disabled at this point! */
10944	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10945	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10946	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10947	/* Interrupts are re-enabled at this point! */
10948
10949	/*
10950	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10951	*/
10952	if (RT_SUCCESS(rcRun))
10953	{ /* very likely */ }
10954	else
10955	{
10956	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10957	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10958	return rcRun;
10959	}
10960
10961	/*
10962	* Profile the VM-exit.
10963	*/
10964	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10965	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10966	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10967	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10968	HMVMX_START_EXIT_DISPATCH_PROF();
10969
10970	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10971
10972	/*
10973	* Handle the VM-exit.
10974	*/
10975	#ifdef HMVMX_USE_FUNCTION_TABLE
10976	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10977	#else
10978	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10979	#endif
10980	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10981	if (rcStrict == VINF_SUCCESS)
10982	{
10983	if (++(*pcLoops) <= cMaxResumeLoops)
10984	continue;
10985	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10986	rcStrict = VINF_EM_RAW_INTERRUPT;
10987	}
10988	break;
10989	}
10990
10991	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10992	return rcStrict;
10993	}
10994
10995
10996	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10997	/**
10998	* Runs the nested-guest code using hardware-assisted VMX.
10999	*
11000	* @returns VBox status code.
11001	* @param pVCpu The cross context virtual CPU structure.
11002	* @param pcLoops Pointer to the number of executed loops.
11003	*
11004	* @sa hmR0VmxRunGuestCodeNormal.
11005	*/
11006	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
11007	{
11008	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11009	Assert(pcLoops);
11010	Assert(*pcLoops <= cMaxResumeLoops);
11011	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11012
11013	/*
11014	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11015	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11016	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11017	*/
11018	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11019	{
11020	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11021	if (RT_SUCCESS(rc))
11022	{ /* likely */ }
11023	else
11024	{
11025	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11026	return rc;
11027	}
11028	}
11029
11030	VMXTRANSIENT VmxTransient;
11031	RT_ZERO(VmxTransient);
11032	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11033	VmxTransient.fIsNestedGuest = true;
11034
11035	/* Paranoia. */
11036	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11037
11038	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11039	for (;;)
11040	{
11041	Assert(!HMR0SuspendPending());
11042	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11043	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11044
11045	/*
11046	* Preparatory work for running guest code, this may force us to
11047	* return to ring-3.
11048	*
11049	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11050	*/
11051	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11052	if (rcStrict != VINF_SUCCESS)
11053	break;
11054
11055	/* Interrupts are disabled at this point! */
11056	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11057	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11058	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11059	/* Interrupts are re-enabled at this point! */
11060
11061	/*
11062	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11063	*/
11064	if (RT_SUCCESS(rcRun))
11065	{ /* very likely */ }
11066	else
11067	{
11068	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11069	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11070	return rcRun;
11071	}
11072
11073	/*
11074	* Profile the VM-exit.
11075	*/
11076	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11077	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11078	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11079	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11080	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11081	HMVMX_START_EXIT_DISPATCH_PROF();
11082
11083	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11084
11085	/*
11086	* Handle the VM-exit.
11087	*/
11088	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11089	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11090	if (rcStrict == VINF_SUCCESS)
11091	{
11092	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11093	{
11094	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11095	rcStrict = VINF_VMX_VMEXIT;
11096	}
11097	else
11098	{
11099	if (++(*pcLoops) <= cMaxResumeLoops)
11100	continue;
11101	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11102	rcStrict = VINF_EM_RAW_INTERRUPT;
11103	}
11104	}
11105	else
11106	Assert(rcStrict != VINF_VMX_VMEXIT);
11107	break;
11108	}
11109
11110	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11111	return rcStrict;
11112	}
11113	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11114
11115
11116	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11117	* probes.
11118	*
11119	* The following few functions and associated structure contains the bloat
11120	* necessary for providing detailed debug events and dtrace probes as well as
11121	* reliable host side single stepping. This works on the principle of
11122	* "subclassing" the normal execution loop and workers. We replace the loop
11123	* method completely and override selected helpers to add necessary adjustments
11124	* to their core operation.
11125	*
11126	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11127	* any performance for debug and analysis features.
11128	*
11129	* @{
11130	*/
11131
11132	/**
11133	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11134	* the debug run loop.
11135	*/
11136	typedef struct VMXRUNDBGSTATE
11137	{
11138	/** The RIP we started executing at. This is for detecting that we stepped. */
11139	uint64_t uRipStart;
11140	/** The CS we started executing with. */
11141	uint16_t uCsStart;
11142
11143	/** Whether we've actually modified the 1st execution control field. */
11144	bool fModifiedProcCtls : 1;
11145	/** Whether we've actually modified the 2nd execution control field. */
11146	bool fModifiedProcCtls2 : 1;
11147	/** Whether we've actually modified the exception bitmap. */
11148	bool fModifiedXcptBitmap : 1;
11149
11150	/** We desire the modified the CR0 mask to be cleared. */
11151	bool fClearCr0Mask : 1;
11152	/** We desire the modified the CR4 mask to be cleared. */
11153	bool fClearCr4Mask : 1;
11154	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11155	uint32_t fCpe1Extra;
11156	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11157	uint32_t fCpe1Unwanted;
11158	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11159	uint32_t fCpe2Extra;
11160	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11161	uint32_t bmXcptExtra;
11162	/** The sequence number of the Dtrace provider settings the state was
11163	* configured against. */
11164	uint32_t uDtraceSettingsSeqNo;
11165	/** VM-exits to check (one bit per VM-exit). */
11166	uint32_t bmExitsToCheck[3];
11167
11168	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11169	uint32_t fProcCtlsInitial;
11170	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11171	uint32_t fProcCtls2Initial;
11172	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11173	uint32_t bmXcptInitial;
11174	} VMXRUNDBGSTATE;
11175	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11176	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11177
11178
11179	/**
11180	* Initializes the VMXRUNDBGSTATE structure.
11181	*
11182	* @param pVCpu The cross context virtual CPU structure of the
11183	* calling EMT.
11184	* @param pVmxTransient The VMX-transient structure.
11185	* @param pDbgState The debug state to initialize.
11186	*/
11187	static void hmR0VmxRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11188	{
11189	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11190	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11191
11192	pDbgState->fModifiedProcCtls = false;
11193	pDbgState->fModifiedProcCtls2 = false;
11194	pDbgState->fModifiedXcptBitmap = false;
11195	pDbgState->fClearCr0Mask = false;
11196	pDbgState->fClearCr4Mask = false;
11197	pDbgState->fCpe1Extra = 0;
11198	pDbgState->fCpe1Unwanted = 0;
11199	pDbgState->fCpe2Extra = 0;
11200	pDbgState->bmXcptExtra = 0;
11201	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11202	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11203	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11204	}
11205
11206
11207	/**
11208	* Updates the VMSC fields with changes requested by @a pDbgState.
11209	*
11210	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11211	* immediately before executing guest code, i.e. when interrupts are disabled.
11212	* We don't check status codes here as we cannot easily assert or return in the
11213	* latter case.
11214	*
11215	* @param pVCpu The cross context virtual CPU structure.
11216	* @param pVmxTransient The VMX-transient structure.
11217	* @param pDbgState The debug state.
11218	*/
11219	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11220	{
11221	/*
11222	* Ensure desired flags in VMCS control fields are set.
11223	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11224	*
11225	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11226	* there should be no stale data in pCtx at this point.
11227	*/
11228	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11229	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11230	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11231	{
11232	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11233	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11234	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11235	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11236	pDbgState->fModifiedProcCtls = true;
11237	}
11238
11239	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11240	{
11241	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11242	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11243	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11244	pDbgState->fModifiedProcCtls2 = true;
11245	}
11246
11247	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11248	{
11249	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11250	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11251	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11252	pDbgState->fModifiedXcptBitmap = true;
11253	}
11254
11255	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11256	{
11257	pVmcsInfo->u64Cr0Mask = 0;
11258	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, 0);
11259	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11260	}
11261
11262	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11263	{
11264	pVmcsInfo->u64Cr4Mask = 0;
11265	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, 0);
11266	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11267	}
11268
11269	NOREF(pVCpu);
11270	}
11271
11272
11273	/**
11274	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11275	* re-entry next time around.
11276	*
11277	* @returns Strict VBox status code (i.e. informational status codes too).
11278	* @param pVCpu The cross context virtual CPU structure.
11279	* @param pVmxTransient The VMX-transient structure.
11280	* @param pDbgState The debug state.
11281	* @param rcStrict The return code from executing the guest using single
11282	* stepping.
11283	*/
11284	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11285	VBOXSTRICTRC rcStrict)
11286	{
11287	/*
11288	* Restore VM-exit control settings as we may not reenter this function the
11289	* next time around.
11290	*/
11291	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11292
11293	/* We reload the initial value, trigger what we can of recalculations the
11294	next time around. From the looks of things, that's all that's required atm. */
11295	if (pDbgState->fModifiedProcCtls)
11296	{
11297	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11298	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11299	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11300	AssertRC(rc2);
11301	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11302	}
11303
11304	/* We're currently the only ones messing with this one, so just restore the
11305	cached value and reload the field. */
11306	if ( pDbgState->fModifiedProcCtls2
11307	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11308	{
11309	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11310	AssertRC(rc2);
11311	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11312	}
11313
11314	/* If we've modified the exception bitmap, we restore it and trigger
11315	reloading and partial recalculation the next time around. */
11316	if (pDbgState->fModifiedXcptBitmap)
11317	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11318
11319	return rcStrict;
11320	}
11321
11322
11323	/**
11324	* Configures VM-exit controls for current DBGF and DTrace settings.
11325	*
11326	* This updates @a pDbgState and the VMCS execution control fields to reflect
11327	* the necessary VM-exits demanded by DBGF and DTrace.
11328	*
11329	* @param pVCpu The cross context virtual CPU structure.
11330	* @param pVmxTransient The VMX-transient structure. May update
11331	* fUpdatedTscOffsettingAndPreemptTimer.
11332	* @param pDbgState The debug state.
11333	*/
11334	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11335	{
11336	/*
11337	* Take down the dtrace serial number so we can spot changes.
11338	*/
11339	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11340	ASMCompilerBarrier();
11341
11342	/*
11343	* We'll rebuild most of the middle block of data members (holding the
11344	* current settings) as we go along here, so start by clearing it all.
11345	*/
11346	pDbgState->bmXcptExtra = 0;
11347	pDbgState->fCpe1Extra = 0;
11348	pDbgState->fCpe1Unwanted = 0;
11349	pDbgState->fCpe2Extra = 0;
11350	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11351	pDbgState->bmExitsToCheck[i] = 0;
11352
11353	/*
11354	* Software interrupts (INT XXh) - no idea how to trigger these...
11355	*/
11356	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11357	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11358	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11359	{
11360	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11361	}
11362
11363	/*
11364	* INT3 breakpoints - triggered by #BP exceptions.
11365	*/
11366	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11367	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11368
11369	/*
11370	* Exception bitmap and XCPT events+probes.
11371	*/
11372	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11373	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11374	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11375
11376	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11377	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11378	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11379	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11380	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11381	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11382	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11383	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11384	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11385	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11386	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11387	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11388	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11389	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11390	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11391	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11392	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11393	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11394
11395	if (pDbgState->bmXcptExtra)
11396	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11397
11398	/*
11399	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11400	*
11401	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11402	* So, when adding/changing/removing please don't forget to update it.
11403	*
11404	* Some of the macros are picking up local variables to save horizontal space,
11405	* (being able to see it in a table is the lesser evil here).
11406	*/
11407	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11408	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11409	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11410	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11411	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11412	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11413	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11414	} else do { } while (0)
11415	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11416	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11417	{ \
11418	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11419	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11420	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11421	} else do { } while (0)
11422	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11423	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11424	{ \
11425	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11426	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11427	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11428	} else do { } while (0)
11429	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11430	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11431	{ \
11432	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11433	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11434	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11435	} else do { } while (0)
11436
11437	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11438	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11439	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11440	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11441	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11442
11443	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11444	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11445	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11446	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11447	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11448	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11449	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11450	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11451	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11452	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11453	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11454	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11455	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11456	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11457	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11458	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11459	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11460	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11461	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11462	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11463	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11464	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11465	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11466	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11467	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11468	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11469	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11470	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11471	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11472	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11473	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11474	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11475	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11476	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11477	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11478	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11479
11480	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11481	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11482	{
11483	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11484	\| CPUMCTX_EXTRN_APIC_TPR);
11485	AssertRC(rc);
11486
11487	#if 0 /** @todo fix me */
11488	pDbgState->fClearCr0Mask = true;
11489	pDbgState->fClearCr4Mask = true;
11490	#endif
11491	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11492	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11493	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11494	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11495	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11496	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11497	require clearing here and in the loop if we start using it. */
11498	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11499	}
11500	else
11501	{
11502	if (pDbgState->fClearCr0Mask)
11503	{
11504	pDbgState->fClearCr0Mask = false;
11505	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11506	}
11507	if (pDbgState->fClearCr4Mask)
11508	{
11509	pDbgState->fClearCr4Mask = false;
11510	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11511	}
11512	}
11513	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11514	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11515
11516	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11517	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11518	{
11519	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11520	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11521	}
11522	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11523	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11524
11525	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11526	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11527	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11528	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11529	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11530	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11531	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11532	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11533	#if 0 /** @todo too slow, fix handler. */
11534	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11535	#endif
11536	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11537
11538	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11539	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11540	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11541	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11542	{
11543	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11544	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11545	}
11546	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11547	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11548	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11549	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11550
11551	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11552	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11553	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11554	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11555	{
11556	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11557	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11558	}
11559	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11560	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11561	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11562	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11563
11564	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11565	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11566	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11567	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11568	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11569	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11570	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11571	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11572	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11573	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11574	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11575	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11576	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11577	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11578	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11579	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11580	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11581	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11582	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11583	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11584	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11585	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11586
11587	#undef IS_EITHER_ENABLED
11588	#undef SET_ONLY_XBM_IF_EITHER_EN
11589	#undef SET_CPE1_XBM_IF_EITHER_EN
11590	#undef SET_CPEU_XBM_IF_EITHER_EN
11591	#undef SET_CPE2_XBM_IF_EITHER_EN
11592
11593	/*
11594	* Sanitize the control stuff.
11595	*/
11596	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11597	if (pDbgState->fCpe2Extra)
11598	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11599	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11600	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11601	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11602	{
11603	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11604	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11605	}
11606
11607	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11608	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11609	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11610	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11611	}
11612
11613
11614	/**
11615	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11616	* appropriate.
11617	*
11618	* The caller has checked the VM-exit against the
11619	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11620	* already, so we don't have to do that either.
11621	*
11622	* @returns Strict VBox status code (i.e. informational status codes too).
11623	* @param pVCpu The cross context virtual CPU structure.
11624	* @param pVmxTransient The VMX-transient structure.
11625	* @param uExitReason The VM-exit reason.
11626	*
11627	* @remarks The name of this function is displayed by dtrace, so keep it short
11628	* and to the point. No longer than 33 chars long, please.
11629	*/
11630	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11631	{
11632	/*
11633	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11634	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11635	*
11636	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11637	* does. Must add/change/remove both places. Same ordering, please.
11638	*
11639	* Added/removed events must also be reflected in the next section
11640	* where we dispatch dtrace events.
11641	*/
11642	bool fDtrace1 = false;
11643	bool fDtrace2 = false;
11644	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11645	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11646	uint32_t uEventArg = 0;
11647	#define SET_EXIT(a_EventSubName) \
11648	do { \
11649	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11650	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11651	} while (0)
11652	#define SET_BOTH(a_EventSubName) \
11653	do { \
11654	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11655	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11656	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11657	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11658	} while (0)
11659	switch (uExitReason)
11660	{
11661	case VMX_EXIT_MTF:
11662	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11663
11664	case VMX_EXIT_XCPT_OR_NMI:
11665	{
11666	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11667	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11668	{
11669	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11670	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11671	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11672	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11673	{
11674	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11675	{
11676	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11677	uEventArg = pVmxTransient->uExitIntErrorCode;
11678	}
11679	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11680	switch (enmEvent1)
11681	{
11682	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11683	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11684	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11685	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11686	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11687	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11688	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11689	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11690	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11691	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11692	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11693	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11694	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11695	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11696	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11697	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11698	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11699	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11700	default: break;
11701	}
11702	}
11703	else
11704	AssertFailed();
11705	break;
11706
11707	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11708	uEventArg = idxVector;
11709	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11710	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11711	break;
11712	}
11713	break;
11714	}
11715
11716	case VMX_EXIT_TRIPLE_FAULT:
11717	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11718	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11719	break;
11720	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11721	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11722	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11723	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11724	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11725
11726	/* Instruction specific VM-exits: */
11727	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11728	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11729	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11730	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11731	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11732	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11733	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11734	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11735	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11736	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11737	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11738	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11739	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11740	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11741	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11742	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11743	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11744	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11745	case VMX_EXIT_MOV_CRX:
11746	hmR0VmxReadExitQualVmcs(pVmxTransient);
11747	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11748	SET_BOTH(CRX_READ);
11749	else
11750	SET_BOTH(CRX_WRITE);
11751	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11752	break;
11753	case VMX_EXIT_MOV_DRX:
11754	hmR0VmxReadExitQualVmcs(pVmxTransient);
11755	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11756	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11757	SET_BOTH(DRX_READ);
11758	else
11759	SET_BOTH(DRX_WRITE);
11760	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11761	break;
11762	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11763	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11764	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11765	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11766	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11767	case VMX_EXIT_GDTR_IDTR_ACCESS:
11768	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11769	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11770	{
11771	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11772	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11773	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11774	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11775	}
11776	break;
11777
11778	case VMX_EXIT_LDTR_TR_ACCESS:
11779	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11780	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11781	{
11782	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11783	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11784	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11785	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11786	}
11787	break;
11788
11789	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11790	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11791	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11792	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11793	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11794	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11795	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11796	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11797	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11798	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11799	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11800
11801	/* Events that aren't relevant at this point. */
11802	case VMX_EXIT_EXT_INT:
11803	case VMX_EXIT_INT_WINDOW:
11804	case VMX_EXIT_NMI_WINDOW:
11805	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11806	case VMX_EXIT_PREEMPT_TIMER:
11807	case VMX_EXIT_IO_INSTR:
11808	break;
11809
11810	/* Errors and unexpected events. */
11811	case VMX_EXIT_INIT_SIGNAL:
11812	case VMX_EXIT_SIPI:
11813	case VMX_EXIT_IO_SMI:
11814	case VMX_EXIT_SMI:
11815	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11816	case VMX_EXIT_ERR_MSR_LOAD:
11817	case VMX_EXIT_ERR_MACHINE_CHECK:
11818	case VMX_EXIT_PML_FULL:
11819	case VMX_EXIT_VIRTUALIZED_EOI:
11820	break;
11821
11822	default:
11823	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11824	break;
11825	}
11826	#undef SET_BOTH
11827	#undef SET_EXIT
11828
11829	/*
11830	* Dtrace tracepoints go first. We do them here at once so we don't
11831	* have to copy the guest state saving and stuff a few dozen times.
11832	* Down side is that we've got to repeat the switch, though this time
11833	* we use enmEvent since the probes are a subset of what DBGF does.
11834	*/
11835	if (fDtrace1 \|\| fDtrace2)
11836	{
11837	hmR0VmxReadExitQualVmcs(pVmxTransient);
11838	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11839	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11840	switch (enmEvent1)
11841	{
11842	/** @todo consider which extra parameters would be helpful for each probe. */
11843	case DBGFEVENT_END: break;
11844	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11845	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11846	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11847	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11848	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11849	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11850	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11851	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11852	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11853	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11854	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11855	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11856	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11857	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11858	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11859	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11860	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11861	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11862	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11863	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11864	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11865	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11866	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11867	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11868	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11869	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11870	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11871	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11872	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11873	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11874	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11875	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11876	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11877	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11878	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11879	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11880	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11881	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11882	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11883	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11884	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11885	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11886	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11887	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11888	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11889	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11890	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11891	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11892	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11893	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11894	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11895	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11896	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11897	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11898	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11899	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11900	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11901	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11902	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11903	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11904	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11905	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11906	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11907	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11908	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11909	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11910	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11911	}
11912	switch (enmEvent2)
11913	{
11914	/** @todo consider which extra parameters would be helpful for each probe. */
11915	case DBGFEVENT_END: break;
11916	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11917	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11918	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11919	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11920	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11921	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11922	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11923	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11924	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11925	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11926	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11927	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11928	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11929	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11930	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11931	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11932	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11933	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11934	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11935	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11936	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11937	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11938	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11939	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11940	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11941	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11942	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11943	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11944	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11945	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11946	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11947	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11948	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11949	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11950	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11951	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11952	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11953	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11954	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11955	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11956	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11957	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11958	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11959	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11960	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11961	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11962	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11963	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11964	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11965	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11966	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11967	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11968	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11969	}
11970	}
11971
11972	/*
11973	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11974	* the DBGF call will do a full check).
11975	*
11976	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11977	* Note! If we have to events, we prioritize the first, i.e. the instruction
11978	* one, in order to avoid event nesting.
11979	*/
11980	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11981	if ( enmEvent1 != DBGFEVENT_END
11982	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11983	{
11984	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11985	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11986	if (rcStrict != VINF_SUCCESS)
11987	return rcStrict;
11988	}
11989	else if ( enmEvent2 != DBGFEVENT_END
11990	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11991	{
11992	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11993	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11994	if (rcStrict != VINF_SUCCESS)
11995	return rcStrict;
11996	}
11997
11998	return VINF_SUCCESS;
11999	}
12000
12001
12002	/**
12003	* Single-stepping VM-exit filtering.
12004	*
12005	* This is preprocessing the VM-exits and deciding whether we've gotten far
12006	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12007	* handling is performed.
12008	*
12009	* @returns Strict VBox status code (i.e. informational status codes too).
12010	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12011	* @param pVmxTransient The VMX-transient structure.
12012	* @param pDbgState The debug state.
12013	*/
12014	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12015	{
12016	/*
12017	* Expensive (saves context) generic dtrace VM-exit probe.
12018	*/
12019	uint32_t const uExitReason = pVmxTransient->uExitReason;
12020	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12021	{ /* more likely */ }
12022	else
12023	{
12024	hmR0VmxReadExitQualVmcs(pVmxTransient);
12025	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12026	AssertRC(rc);
12027	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12028	}
12029
12030	/*
12031	* Check for host NMI, just to get that out of the way.
12032	*/
12033	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12034	{ /* normally likely */ }
12035	else
12036	{
12037	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12038	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12039	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12040	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12041	}
12042
12043	/*
12044	* Check for single stepping event if we're stepping.
12045	*/
12046	if (pVCpu->hm.s.fSingleInstruction)
12047	{
12048	switch (uExitReason)
12049	{
12050	case VMX_EXIT_MTF:
12051	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12052
12053	/* Various events: */
12054	case VMX_EXIT_XCPT_OR_NMI:
12055	case VMX_EXIT_EXT_INT:
12056	case VMX_EXIT_TRIPLE_FAULT:
12057	case VMX_EXIT_INT_WINDOW:
12058	case VMX_EXIT_NMI_WINDOW:
12059	case VMX_EXIT_TASK_SWITCH:
12060	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12061	case VMX_EXIT_APIC_ACCESS:
12062	case VMX_EXIT_EPT_VIOLATION:
12063	case VMX_EXIT_EPT_MISCONFIG:
12064	case VMX_EXIT_PREEMPT_TIMER:
12065
12066	/* Instruction specific VM-exits: */
12067	case VMX_EXIT_CPUID:
12068	case VMX_EXIT_GETSEC:
12069	case VMX_EXIT_HLT:
12070	case VMX_EXIT_INVD:
12071	case VMX_EXIT_INVLPG:
12072	case VMX_EXIT_RDPMC:
12073	case VMX_EXIT_RDTSC:
12074	case VMX_EXIT_RSM:
12075	case VMX_EXIT_VMCALL:
12076	case VMX_EXIT_VMCLEAR:
12077	case VMX_EXIT_VMLAUNCH:
12078	case VMX_EXIT_VMPTRLD:
12079	case VMX_EXIT_VMPTRST:
12080	case VMX_EXIT_VMREAD:
12081	case VMX_EXIT_VMRESUME:
12082	case VMX_EXIT_VMWRITE:
12083	case VMX_EXIT_VMXOFF:
12084	case VMX_EXIT_VMXON:
12085	case VMX_EXIT_MOV_CRX:
12086	case VMX_EXIT_MOV_DRX:
12087	case VMX_EXIT_IO_INSTR:
12088	case VMX_EXIT_RDMSR:
12089	case VMX_EXIT_WRMSR:
12090	case VMX_EXIT_MWAIT:
12091	case VMX_EXIT_MONITOR:
12092	case VMX_EXIT_PAUSE:
12093	case VMX_EXIT_GDTR_IDTR_ACCESS:
12094	case VMX_EXIT_LDTR_TR_ACCESS:
12095	case VMX_EXIT_INVEPT:
12096	case VMX_EXIT_RDTSCP:
12097	case VMX_EXIT_INVVPID:
12098	case VMX_EXIT_WBINVD:
12099	case VMX_EXIT_XSETBV:
12100	case VMX_EXIT_RDRAND:
12101	case VMX_EXIT_INVPCID:
12102	case VMX_EXIT_VMFUNC:
12103	case VMX_EXIT_RDSEED:
12104	case VMX_EXIT_XSAVES:
12105	case VMX_EXIT_XRSTORS:
12106	{
12107	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12108	AssertRCReturn(rc, rc);
12109	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12110	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12111	return VINF_EM_DBG_STEPPED;
12112	break;
12113	}
12114
12115	/* Errors and unexpected events: */
12116	case VMX_EXIT_INIT_SIGNAL:
12117	case VMX_EXIT_SIPI:
12118	case VMX_EXIT_IO_SMI:
12119	case VMX_EXIT_SMI:
12120	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12121	case VMX_EXIT_ERR_MSR_LOAD:
12122	case VMX_EXIT_ERR_MACHINE_CHECK:
12123	case VMX_EXIT_PML_FULL:
12124	case VMX_EXIT_VIRTUALIZED_EOI:
12125	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12126	break;
12127
12128	default:
12129	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12130	break;
12131	}
12132	}
12133
12134	/*
12135	* Check for debugger event breakpoints and dtrace probes.
12136	*/
12137	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12138	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12139	{
12140	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12141	if (rcStrict != VINF_SUCCESS)
12142	return rcStrict;
12143	}
12144
12145	/*
12146	* Normal processing.
12147	*/
12148	#ifdef HMVMX_USE_FUNCTION_TABLE
12149	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12150	#else
12151	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12152	#endif
12153	}
12154
12155
12156	/**
12157	* Single steps guest code using hardware-assisted VMX.
12158	*
12159	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12160	* but single-stepping through the hypervisor debugger.
12161	*
12162	* @returns Strict VBox status code (i.e. informational status codes too).
12163	* @param pVCpu The cross context virtual CPU structure.
12164	* @param pcLoops Pointer to the number of executed loops.
12165	*
12166	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12167	*/
12168	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
12169	{
12170	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12171	Assert(pcLoops);
12172	Assert(*pcLoops <= cMaxResumeLoops);
12173
12174	VMXTRANSIENT VmxTransient;
12175	RT_ZERO(VmxTransient);
12176	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12177
12178	/* Set HMCPU indicators. */
12179	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12180	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12181	pVCpu->hm.s.fDebugWantRdTscExit = false;
12182	pVCpu->hm.s.fUsingDebugLoop = true;
12183
12184	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12185	VMXRUNDBGSTATE DbgState;
12186	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12187	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12188
12189	/*
12190	* The loop.
12191	*/
12192	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12193	for (;;)
12194	{
12195	Assert(!HMR0SuspendPending());
12196	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12197	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12198	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12199
12200	/* Set up VM-execution controls the next two can respond to. */
12201	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12202
12203	/*
12204	* Preparatory work for running guest code, this may force us to
12205	* return to ring-3.
12206	*
12207	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12208	*/
12209	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12210	if (rcStrict != VINF_SUCCESS)
12211	break;
12212
12213	/* Interrupts are disabled at this point! */
12214	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12215
12216	/* Override any obnoxious code in the above two calls. */
12217	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12218
12219	/*
12220	* Finally execute the guest.
12221	*/
12222	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12223
12224	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12225	/* Interrupts are re-enabled at this point! */
12226
12227	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12228	if (RT_SUCCESS(rcRun))
12229	{ /* very likely */ }
12230	else
12231	{
12232	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12233	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12234	return rcRun;
12235	}
12236
12237	/* Profile the VM-exit. */
12238	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12239	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12240	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12241	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12242	HMVMX_START_EXIT_DISPATCH_PROF();
12243
12244	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12245
12246	/*
12247	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12248	*/
12249	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12250	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12251	if (rcStrict != VINF_SUCCESS)
12252	break;
12253	if (++(*pcLoops) > cMaxResumeLoops)
12254	{
12255	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12256	rcStrict = VINF_EM_RAW_INTERRUPT;
12257	break;
12258	}
12259
12260	/*
12261	* Stepping: Did the RIP change, if so, consider it a single step.
12262	* Otherwise, make sure one of the TFs gets set.
12263	*/
12264	if (fStepping)
12265	{
12266	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12267	AssertRC(rc);
12268	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12269	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12270	{
12271	rcStrict = VINF_EM_DBG_STEPPED;
12272	break;
12273	}
12274	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12275	}
12276
12277	/*
12278	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12279	*/
12280	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12281	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12282	}
12283
12284	/*
12285	* Clear the X86_EFL_TF if necessary.
12286	*/
12287	if (pVCpu->hm.s.fClearTrapFlag)
12288	{
12289	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12290	AssertRC(rc);
12291	pVCpu->hm.s.fClearTrapFlag = false;
12292	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12293	}
12294	/** @todo there seems to be issues with the resume flag when the monitor trap
12295	* flag is pending without being used. Seen early in bios init when
12296	* accessing APIC page in protected mode. */
12297
12298	/*
12299	* Restore VM-exit control settings as we may not re-enter this function the
12300	* next time around.
12301	*/
12302	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12303
12304	/* Restore HMCPU indicators. */
12305	pVCpu->hm.s.fUsingDebugLoop = false;
12306	pVCpu->hm.s.fDebugWantRdTscExit = false;
12307	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12308
12309	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12310	return rcStrict;
12311	}
12312
12313
12314	/** @} */
12315
12316
12317	/**
12318	* Checks if any expensive dtrace probes are enabled and we should go to the
12319	* debug loop.
12320	*
12321	* @returns true if we should use debug loop, false if not.
12322	*/
12323	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12324	{
12325	/* It's probably faster to OR the raw 32-bit counter variables together.
12326	Since the variables are in an array and the probes are next to one
12327	another (more or less), we have good locality. So, better read
12328	eight-nine cache lines ever time and only have one conditional, than
12329	128+ conditionals, right? */
12330	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12331	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12332	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12333	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12334	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12335	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12336	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12337	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12338	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12339	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12340	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12341	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12342	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12343	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12344	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12345	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12346	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12347	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12348	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12349	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12350	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12351	) != 0
12352	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12353	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12354	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12355	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12356	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12357	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12358	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12359	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12360	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12361	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12362	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12363	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12364	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12365	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12366	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12367	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12368	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12369	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12370	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12371	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12372	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12373	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12374	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12375	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12376	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12377	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12378	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12379	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12380	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12381	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12382	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12383	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12384	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12385	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12386	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12387	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12388	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12389	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12390	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12391	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12392	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12393	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12394	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12395	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12396	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12397	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12398	) != 0
12399	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12400	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12401	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12402	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12403	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12404	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12405	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12406	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12407	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12408	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12409	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12410	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12411	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12412	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12413	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12414	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12415	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12416	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12417	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12418	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12419	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12420	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12421	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12422	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12423	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12424	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12425	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12426	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12427	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12428	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12429	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12430	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12431	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12432	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12433	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12434	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12435	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12436	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12437	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12438	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12439	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12440	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12441	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12442	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12443	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12444	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12445	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12446	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12447	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12448	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12449	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12450	) != 0;
12451	}
12452
12453
12454	/**
12455	* Runs the guest using hardware-assisted VMX.
12456	*
12457	* @returns Strict VBox status code (i.e. informational status codes too).
12458	* @param pVCpu The cross context virtual CPU structure.
12459	*/
12460	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu)
12461	{
12462	AssertPtr(pVCpu);
12463	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12464	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12465	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12466	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12467
12468	VBOXSTRICTRC rcStrict;
12469	uint32_t cLoops = 0;
12470	for (;;)
12471	{
12472	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12473	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12474	#else
12475	NOREF(pCtx);
12476	bool const fInNestedGuestMode = false;
12477	#endif
12478	if (!fInNestedGuestMode)
12479	{
12480	if ( !pVCpu->hm.s.fUseDebugLoop
12481	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12482	&& !DBGFIsStepping(pVCpu)
12483	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12484	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12485	else
12486	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12487	}
12488	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12489	else
12490	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12491
12492	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12493	{
12494	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12495	continue;
12496	}
12497	if (rcStrict == VINF_VMX_VMEXIT)
12498	{
12499	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12500	continue;
12501	}
12502	#endif
12503	break;
12504	}
12505
12506	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12507	switch (rcLoop)
12508	{
12509	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12510	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12511	}
12512
12513	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12514	if (RT_FAILURE(rc2))
12515	{
12516	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12517	rcStrict = rc2;
12518	}
12519	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12520	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12521	return rcStrict;
12522	}
12523
12524
12525	#ifndef HMVMX_USE_FUNCTION_TABLE
12526	/**
12527	* Handles a guest VM-exit from hardware-assisted VMX execution.
12528	*
12529	* @returns Strict VBox status code (i.e. informational status codes too).
12530	* @param pVCpu The cross context virtual CPU structure.
12531	* @param pVmxTransient The VMX-transient structure.
12532	*/
12533	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12534	{
12535	#ifdef DEBUG_ramshankar
12536	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12537	do { \
12538	if (a_fSave != 0) \
12539	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12540	VBOXSTRICTRC rcStrict = a_CallExpr; \
12541	if (a_fSave != 0) \
12542	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12543	return rcStrict; \
12544	} while (0)
12545	#else
12546	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12547	#endif
12548	uint32_t const uExitReason = pVmxTransient->uExitReason;
12549	switch (uExitReason)
12550	{
12551	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12552	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12553	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12554	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12555	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12556	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12557	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12558	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12559	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12560	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12561	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12562	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12563	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12564	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12565	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12566	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12567	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12568	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12569	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12570	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12571	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12572	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12573	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12574	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12575	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12576	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12577	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12578	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12579	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12580	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12581	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12582	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12583	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12584	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12585	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12586	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12587	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12588	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12589	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12590	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12591	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12592	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12593	#else
12594	case VMX_EXIT_VMCLEAR:
12595	case VMX_EXIT_VMLAUNCH:
12596	case VMX_EXIT_VMPTRLD:
12597	case VMX_EXIT_VMPTRST:
12598	case VMX_EXIT_VMREAD:
12599	case VMX_EXIT_VMRESUME:
12600	case VMX_EXIT_VMWRITE:
12601	case VMX_EXIT_VMXOFF:
12602	case VMX_EXIT_VMXON:
12603	case VMX_EXIT_INVVPID:
12604	case VMX_EXIT_INVEPT:
12605	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12606	#endif
12607
12608	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12609	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12610	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12611
12612	case VMX_EXIT_INIT_SIGNAL:
12613	case VMX_EXIT_SIPI:
12614	case VMX_EXIT_IO_SMI:
12615	case VMX_EXIT_SMI:
12616	case VMX_EXIT_ERR_MSR_LOAD:
12617	case VMX_EXIT_ERR_MACHINE_CHECK:
12618	case VMX_EXIT_PML_FULL:
12619	case VMX_EXIT_VIRTUALIZED_EOI:
12620	case VMX_EXIT_GDTR_IDTR_ACCESS:
12621	case VMX_EXIT_LDTR_TR_ACCESS:
12622	case VMX_EXIT_APIC_WRITE:
12623	case VMX_EXIT_RDRAND:
12624	case VMX_EXIT_RSM:
12625	case VMX_EXIT_VMFUNC:
12626	case VMX_EXIT_ENCLS:
12627	case VMX_EXIT_RDSEED:
12628	case VMX_EXIT_XSAVES:
12629	case VMX_EXIT_XRSTORS:
12630	case VMX_EXIT_UMWAIT:
12631	case VMX_EXIT_TPAUSE:
12632	default:
12633	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12634	}
12635	#undef VMEXIT_CALL_RET
12636	}
12637	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12638
12639
12640	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12641	/**
12642	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12643	*
12644	* @returns Strict VBox status code (i.e. informational status codes too).
12645	* @param pVCpu The cross context virtual CPU structure.
12646	* @param pVmxTransient The VMX-transient structure.
12647	*/
12648	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12649	{
12650	/** @todo NSTVMX: Remove after debugging page-fault issue. */
12651	#ifdef DEBUG_ramshankar
12652	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12653	Log4Func(("cs:rip=%#04x:%#RX64 rsp=%#RX64 eflags=%#RX32 cr3=%#RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12654	pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.cr3));
12655	#endif
12656
12657	uint32_t const uExitReason = pVmxTransient->uExitReason;
12658	switch (uExitReason)
12659	{
12660	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12661	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12662	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12663	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12664	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12665
12666	/*
12667	* We shouldn't direct host physical interrupts to the nested-guest.
12668	*/
12669	case VMX_EXIT_EXT_INT:
12670	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12671
12672	/*
12673	* Instructions that cause VM-exits unconditionally or the condition is
12674	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12675	* happens, it's guaranteed to be a nested-guest VM-exit).
12676	*
12677	* - Provides VM-exit instruction length ONLY.
12678	*/
12679	case VMX_EXIT_CPUID: /* Unconditional. */
12680	case VMX_EXIT_VMCALL:
12681	case VMX_EXIT_GETSEC:
12682	case VMX_EXIT_INVD:
12683	case VMX_EXIT_XSETBV:
12684	case VMX_EXIT_VMLAUNCH:
12685	case VMX_EXIT_VMRESUME:
12686	case VMX_EXIT_VMXOFF:
12687	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
12688	case VMX_EXIT_VMFUNC:
12689	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12690
12691	/*
12692	* Instructions that cause VM-exits unconditionally or the condition is
12693	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12694	* happens, it's guaranteed to be a nested-guest VM-exit).
12695	*
12696	* - Provides VM-exit instruction length.
12697	* - Provides VM-exit information.
12698	* - Optionally provides Exit qualification.
12699	*
12700	* Since Exit qualification is 0 for all VM-exits where it is not
12701	* applicable, reading and passing it to the guest should produce
12702	* defined behavior.
12703	*
12704	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12705	*/
12706	case VMX_EXIT_INVEPT: /* Unconditional. */
12707	case VMX_EXIT_INVVPID:
12708	case VMX_EXIT_VMCLEAR:
12709	case VMX_EXIT_VMPTRLD:
12710	case VMX_EXIT_VMPTRST:
12711	case VMX_EXIT_VMXON:
12712	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
12713	case VMX_EXIT_LDTR_TR_ACCESS:
12714	case VMX_EXIT_RDRAND:
12715	case VMX_EXIT_RDSEED:
12716	case VMX_EXIT_XSAVES:
12717	case VMX_EXIT_XRSTORS:
12718	case VMX_EXIT_UMWAIT:
12719	case VMX_EXIT_TPAUSE:
12720	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12721
12722	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12723	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12724	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12725	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12726	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12727	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12728	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12729	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12730	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12731	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12732	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12733	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12734	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12735	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12736	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12737	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12738	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12739	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12740	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12741
12742	case VMX_EXIT_PREEMPT_TIMER:
12743	{
12744	/** @todo NSTVMX: Preempt timer. */
12745	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12746	}
12747
12748	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12749	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12750
12751	case VMX_EXIT_VMREAD:
12752	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12753
12754	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12755	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12756
12757	case VMX_EXIT_INIT_SIGNAL:
12758	case VMX_EXIT_SIPI:
12759	case VMX_EXIT_IO_SMI:
12760	case VMX_EXIT_SMI:
12761	case VMX_EXIT_ERR_MSR_LOAD:
12762	case VMX_EXIT_ERR_MACHINE_CHECK:
12763	case VMX_EXIT_PML_FULL:
12764	case VMX_EXIT_RSM:
12765	default:
12766	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12767	}
12768	}
12769	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12770
12771
12772	/** @name VM-exit helpers.
12773	* @{
12774	*/
12775	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12776	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12777	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12778
12779	/** Macro for VM-exits called unexpectedly. */
12780	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
12781	do { \
12782	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
12783	return VERR_VMX_UNEXPECTED_EXIT; \
12784	} while (0)
12785
12786	#ifdef VBOX_STRICT
12787	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12788	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12789	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12790
12791	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12792	do { \
12793	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12794	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12795	} while (0)
12796
12797	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12798	do { \
12799	AssertPtr((a_pVCpu)); \
12800	AssertPtr((a_pVmxTransient)); \
12801	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12802	Assert((a_pVmxTransient)->pVmcsInfo); \
12803	Assert(ASMIntAreEnabled()); \
12804	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12805	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12806	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
12807	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12808	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12809	HMVMX_ASSERT_PREEMPT_CPUID(); \
12810	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12811	} while (0)
12812
12813	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12814	do { \
12815	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12816	Assert((a_pVmxTransient)->fIsNestedGuest); \
12817	} while (0)
12818
12819	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12820	do { \
12821	Log4Func(("\n")); \
12822	} while (0)
12823	#else
12824	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12825	do { \
12826	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12827	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12828	} while (0)
12829
12830	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12831	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12832
12833	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12834	#endif
12835
12836	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12837	/** Macro that does the necessary privilege checks and intercepted VM-exits for
12838	* guests that attempted to execute a VMX instruction. */
12839	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
12840	do \
12841	{ \
12842	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
12843	if (rcStrictTmp == VINF_SUCCESS) \
12844	{ /* likely */ } \
12845	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12846	{ \
12847	Assert((a_pVCpu)->hm.s.Event.fPending); \
12848	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
12849	return VINF_SUCCESS; \
12850	} \
12851	else \
12852	{ \
12853	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
12854	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
12855	} \
12856	} while (0)
12857
12858	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
12859	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
12860	do \
12861	{ \
12862	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
12863	(a_pGCPtrEffAddr)); \
12864	if (rcStrictTmp == VINF_SUCCESS) \
12865	{ /* likely */ } \
12866	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12867	{ \
12868	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
12869	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
12870	NOREF(uXcptTmp); \
12871	return VINF_SUCCESS; \
12872	} \
12873	else \
12874	{ \
12875	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
12876	return rcStrictTmp; \
12877	} \
12878	} while (0)
12879	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12880
12881
12882	/**
12883	* Advances the guest RIP by the specified number of bytes.
12884	*
12885	* @param pVCpu The cross context virtual CPU structure.
12886	* @param cbInstr Number of bytes to advance the RIP by.
12887	*
12888	* @remarks No-long-jump zone!!!
12889	*/
12890	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
12891	{
12892	/* Advance the RIP. */
12893	pVCpu->cpum.GstCtx.rip += cbInstr;
12894	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12895
12896	/* Update interrupt inhibition. */
12897	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12898	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12899	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12900	}
12901
12902
12903	/**
12904	* Advances the guest RIP after reading it from the VMCS.
12905	*
12906	* @returns VBox status code, no informational status codes.
12907	* @param pVCpu The cross context virtual CPU structure.
12908	* @param pVmxTransient The VMX-transient structure.
12909	*
12910	* @remarks No-long-jump zone!!!
12911	*/
12912	static int hmR0VmxAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12913	{
12914	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12915	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12916	AssertRCReturn(rc, rc);
12917
12918	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
12919	return VINF_SUCCESS;
12920	}
12921
12922
12923	/**
12924	* Handle a condition that occurred while delivering an event through the guest or
12925	* nested-guest IDT.
12926	*
12927	* @returns Strict VBox status code (i.e. informational status codes too).
12928	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12929	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12930	* to continue execution of the guest which will delivery the \#DF.
12931	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12932	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12933	*
12934	* @param pVCpu The cross context virtual CPU structure.
12935	* @param pVmxTransient The VMX-transient structure.
12936	*
12937	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
12938	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
12939	* is due to an EPT violation, PML full or SPP-related event.
12940	*
12941	* @remarks No-long-jump zone!!!
12942	*/
12943	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12944	{
12945	Assert(!pVCpu->hm.s.Event.fPending);
12946	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
12947	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
12948	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
12949	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
12950	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
12951
12952	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12953	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12954	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
12955	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12956	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
12957	{
12958	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
12959	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
12960
12961	/*
12962	* If the event was a software interrupt (generated with INT n) or a software exception
12963	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12964	* can handle the VM-exit and continue guest execution which will re-execute the
12965	* instruction rather than re-injecting the exception, as that can cause premature
12966	* trips to ring-3 before injection and involve TRPM which currently has no way of
12967	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12968	* the problem).
12969	*/
12970	IEMXCPTRAISE enmRaise;
12971	IEMXCPTRAISEINFO fRaiseInfo;
12972	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12973	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12974	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12975	{
12976	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12977	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12978	}
12979	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
12980	{
12981	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
12982	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12983	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
12984
12985	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12986	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12987
12988	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12989
12990	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12991	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12992	{
12993	pVmxTransient->fVectoringPF = true;
12994	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12995	}
12996	}
12997	else
12998	{
12999	/*
13000	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13001	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13002	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13003	*/
13004	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13005	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13006	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13007	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13008	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13009	}
13010
13011	/*
13012	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13013	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13014	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13015	* subsequent VM-entry would fail, see @bugref{7445}.
13016	*
13017	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13018	*/
13019	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13020	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
13021	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13022	&& CPUMIsGuestNmiBlocking(pVCpu))
13023	{
13024	CPUMSetGuestNmiBlocking(pVCpu, false);
13025	}
13026
13027	switch (enmRaise)
13028	{
13029	case IEMXCPTRAISE_CURRENT_XCPT:
13030	{
13031	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
13032	Assert(rcStrict == VINF_SUCCESS);
13033	break;
13034	}
13035
13036	case IEMXCPTRAISE_PREV_EVENT:
13037	{
13038	uint32_t u32ErrCode;
13039	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13040	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13041	else
13042	u32ErrCode = 0;
13043
13044	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13045	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13046	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13047	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13048
13049	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13050	pVCpu->hm.s.Event.u32ErrCode));
13051	Assert(rcStrict == VINF_SUCCESS);
13052	break;
13053	}
13054
13055	case IEMXCPTRAISE_REEXEC_INSTR:
13056	Assert(rcStrict == VINF_SUCCESS);
13057	break;
13058
13059	case IEMXCPTRAISE_DOUBLE_FAULT:
13060	{
13061	/*
13062	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13063	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13064	*/
13065	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13066	{
13067	pVmxTransient->fVectoringDoublePF = true;
13068	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13069	pVCpu->cpum.GstCtx.cr2));
13070	rcStrict = VINF_SUCCESS;
13071	}
13072	else
13073	{
13074	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13075	hmR0VmxSetPendingXcptDF(pVCpu);
13076	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13077	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13078	rcStrict = VINF_HM_DOUBLE_FAULT;
13079	}
13080	break;
13081	}
13082
13083	case IEMXCPTRAISE_TRIPLE_FAULT:
13084	{
13085	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13086	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13087	rcStrict = VINF_EM_RESET;
13088	break;
13089	}
13090
13091	case IEMXCPTRAISE_CPU_HANG:
13092	{
13093	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13094	rcStrict = VERR_EM_GUEST_CPU_HANG;
13095	break;
13096	}
13097
13098	default:
13099	{
13100	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13101	rcStrict = VERR_VMX_IPE_2;
13102	break;
13103	}
13104	}
13105	}
13106	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13107	&& !CPUMIsGuestNmiBlocking(pVCpu))
13108	{
13109	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13110	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13111	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13112	{
13113	/*
13114	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13115	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13116	* that NMIs remain blocked until the IRET execution is completed.
13117	*
13118	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13119	*/
13120	CPUMSetGuestNmiBlocking(pVCpu, true);
13121	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13122	}
13123	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13124	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13125	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13126	{
13127	/*
13128	* Execution of IRET caused an EPT violation, page-modification log-full event or
13129	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13130	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13131	* that NMIs remain blocked until the IRET execution is completed.
13132	*
13133	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13134	*/
13135	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13136	{
13137	CPUMSetGuestNmiBlocking(pVCpu, true);
13138	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13139	}
13140	}
13141	}
13142
13143	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13144	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13145	return rcStrict;
13146	}
13147
13148
13149	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13150	/**
13151	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13152	* guest attempting to execute a VMX instruction.
13153	*
13154	* @returns Strict VBox status code (i.e. informational status codes too).
13155	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13156	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13157	*
13158	* @param pVCpu The cross context virtual CPU structure.
13159	* @param uExitReason The VM-exit reason.
13160	*
13161	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13162	* @remarks No-long-jump zone!!!
13163	*/
13164	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
13165	{
13166	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13167	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13168
13169	/*
13170	* The physical CPU would have already checked the CPU mode/code segment.
13171	* We shall just assert here for paranoia.
13172	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13173	*/
13174	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13175	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13176	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13177
13178	if (uExitReason == VMX_EXIT_VMXON)
13179	{
13180	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13181
13182	/*
13183	* We check CR4.VMXE because it is required to be always set while in VMX operation
13184	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13185	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13186	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13187	*/
13188	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13189	{
13190	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13191	hmR0VmxSetPendingXcptUD(pVCpu);
13192	return VINF_HM_PENDING_XCPT;
13193	}
13194	}
13195	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13196	{
13197	/*
13198	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13199	* (other than VMXON), we need to raise a #UD.
13200	*/
13201	Log4Func(("Not in VMX root mode -> #UD\n"));
13202	hmR0VmxSetPendingXcptUD(pVCpu);
13203	return VINF_HM_PENDING_XCPT;
13204	}
13205
13206	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13207	return VINF_SUCCESS;
13208	}
13209
13210
13211	/**
13212	* Decodes the memory operand of an instruction that caused a VM-exit.
13213	*
13214	* The Exit qualification field provides the displacement field for memory
13215	* operand instructions, if any.
13216	*
13217	* @returns Strict VBox status code (i.e. informational status codes too).
13218	* @retval VINF_SUCCESS if the operand was successfully decoded.
13219	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13220	* operand.
13221	* @param pVCpu The cross context virtual CPU structure.
13222	* @param uExitInstrInfo The VM-exit instruction information field.
13223	* @param enmMemAccess The memory operand's access type (read or write).
13224	* @param GCPtrDisp The instruction displacement field, if any. For
13225	* RIP-relative addressing pass RIP + displacement here.
13226	* @param pGCPtrMem Where to store the effective destination memory address.
13227	*
13228	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13229	* virtual-8086 mode hence skips those checks while verifying if the
13230	* segment is valid.
13231	*/
13232	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13233	PRTGCPTR pGCPtrMem)
13234	{
13235	Assert(pGCPtrMem);
13236	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13237	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13238	\| CPUMCTX_EXTRN_CR0);
13239
13240	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13241	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13242	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13243
13244	VMXEXITINSTRINFO ExitInstrInfo;
13245	ExitInstrInfo.u = uExitInstrInfo;
13246	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13247	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13248	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13249	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13250	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13251	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13252	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13253	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13254	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13255
13256	/*
13257	* Validate instruction information.
13258	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13259	*/
13260	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13261	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13262	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13263	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13264	AssertLogRelMsgReturn(fIsMemOperand,
13265	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13266
13267	/*
13268	* Compute the complete effective address.
13269	*
13270	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13271	* See AMD spec. 4.5.2 "Segment Registers".
13272	*/
13273	RTGCPTR GCPtrMem = GCPtrDisp;
13274	if (fBaseRegValid)
13275	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13276	if (fIdxRegValid)
13277	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13278
13279	RTGCPTR const GCPtrOff = GCPtrMem;
13280	if ( !fIsLongMode
13281	\|\| iSegReg >= X86_SREG_FS)
13282	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13283	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13284
13285	/*
13286	* Validate effective address.
13287	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13288	*/
13289	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13290	Assert(cbAccess > 0);
13291	if (fIsLongMode)
13292	{
13293	if (X86_IS_CANONICAL(GCPtrMem))
13294	{
13295	*pGCPtrMem = GCPtrMem;
13296	return VINF_SUCCESS;
13297	}
13298
13299	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13300	* "Data Limit Checks in 64-bit Mode". */
13301	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13302	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13303	return VINF_HM_PENDING_XCPT;
13304	}
13305
13306	/*
13307	* This is a watered down version of iemMemApplySegment().
13308	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13309	* and segment CPL/DPL checks are skipped.
13310	*/
13311	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13312	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13313	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13314
13315	/* Check if the segment is present and usable. */
13316	if ( pSel->Attr.n.u1Present
13317	&& !pSel->Attr.n.u1Unusable)
13318	{
13319	Assert(pSel->Attr.n.u1DescType);
13320	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13321	{
13322	/* Check permissions for the data segment. */
13323	if ( enmMemAccess == VMXMEMACCESS_WRITE
13324	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13325	{
13326	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13327	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13328	return VINF_HM_PENDING_XCPT;
13329	}
13330
13331	/* Check limits if it's a normal data segment. */
13332	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13333	{
13334	if ( GCPtrFirst32 > pSel->u32Limit
13335	\|\| GCPtrLast32 > pSel->u32Limit)
13336	{
13337	Log4Func(("Data segment limit exceeded. "
13338	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13339	GCPtrLast32, pSel->u32Limit));
13340	if (iSegReg == X86_SREG_SS)
13341	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13342	else
13343	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13344	return VINF_HM_PENDING_XCPT;
13345	}
13346	}
13347	else
13348	{
13349	/* Check limits if it's an expand-down data segment.
13350	Note! The upper boundary is defined by the B bit, not the G bit! */
13351	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13352	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13353	{
13354	Log4Func(("Expand-down data segment limit exceeded. "
13355	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13356	GCPtrLast32, pSel->u32Limit));
13357	if (iSegReg == X86_SREG_SS)
13358	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13359	else
13360	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13361	return VINF_HM_PENDING_XCPT;
13362	}
13363	}
13364	}
13365	else
13366	{
13367	/* Check permissions for the code segment. */
13368	if ( enmMemAccess == VMXMEMACCESS_WRITE
13369	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13370	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13371	{
13372	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13373	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13374	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13375	return VINF_HM_PENDING_XCPT;
13376	}
13377
13378	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13379	if ( GCPtrFirst32 > pSel->u32Limit
13380	\|\| GCPtrLast32 > pSel->u32Limit)
13381	{
13382	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13383	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13384	if (iSegReg == X86_SREG_SS)
13385	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13386	else
13387	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13388	return VINF_HM_PENDING_XCPT;
13389	}
13390	}
13391	}
13392	else
13393	{
13394	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13395	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13396	return VINF_HM_PENDING_XCPT;
13397	}
13398
13399	*pGCPtrMem = GCPtrMem;
13400	return VINF_SUCCESS;
13401	}
13402	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13403
13404
13405	/**
13406	* VM-exit helper for LMSW.
13407	*/
13408	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13409	{
13410	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13411	AssertRCReturn(rc, rc);
13412
13413	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13414	AssertMsg( rcStrict == VINF_SUCCESS
13415	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13416
13417	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13418	if (rcStrict == VINF_IEM_RAISED_XCPT)
13419	{
13420	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13421	rcStrict = VINF_SUCCESS;
13422	}
13423
13424	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13425	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13426	return rcStrict;
13427	}
13428
13429
13430	/**
13431	* VM-exit helper for CLTS.
13432	*/
13433	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13434	{
13435	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13436	AssertRCReturn(rc, rc);
13437
13438	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13439	AssertMsg( rcStrict == VINF_SUCCESS
13440	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13441
13442	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13443	if (rcStrict == VINF_IEM_RAISED_XCPT)
13444	{
13445	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13446	rcStrict = VINF_SUCCESS;
13447	}
13448
13449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13450	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13451	return rcStrict;
13452	}
13453
13454
13455	/**
13456	* VM-exit helper for MOV from CRx (CRx read).
13457	*/
13458	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13459	{
13460	Assert(iCrReg < 16);
13461	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13462
13463	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13464	AssertRCReturn(rc, rc);
13465
13466	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13467	AssertMsg( rcStrict == VINF_SUCCESS
13468	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13469
13470	if (iGReg == X86_GREG_xSP)
13471	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13472	else
13473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13474	#ifdef VBOX_WITH_STATISTICS
13475	switch (iCrReg)
13476	{
13477	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13478	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13479	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13480	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13481	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13482	}
13483	#endif
13484	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13485	return rcStrict;
13486	}
13487
13488
13489	/**
13490	* VM-exit helper for MOV to CRx (CRx write).
13491	*/
13492	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13493	{
13494	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13495	AssertRCReturn(rc, rc);
13496
13497	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13498	AssertMsg( rcStrict == VINF_SUCCESS
13499	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13500	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13501
13502	switch (iCrReg)
13503	{
13504	case 0:
13505	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
13506	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13507	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13508	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13509	break;
13510
13511	case 2:
13512	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13513	/* Nothing to do here, CR2 it's not part of the VMCS. */
13514	break;
13515
13516	case 3:
13517	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13518	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13519	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13520	break;
13521
13522	case 4:
13523	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13524	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13525	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13526	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13527	break;
13528
13529	case 8:
13530	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13531	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13532	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13533	break;
13534
13535	default:
13536	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13537	break;
13538	}
13539
13540	if (rcStrict == VINF_IEM_RAISED_XCPT)
13541	{
13542	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13543	rcStrict = VINF_SUCCESS;
13544	}
13545	return rcStrict;
13546	}
13547
13548
13549	/**
13550	* VM-exit exception handler for \#PF (Page-fault exception).
13551	*
13552	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13553	*/
13554	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13555	{
13556	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13557	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13558	hmR0VmxReadExitQualVmcs(pVmxTransient);
13559
13560	if (!pVM->hm.s.fNestedPaging)
13561	{ /* likely */ }
13562	else
13563	{
13564	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13565	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13566	#endif
13567	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13568	if (!pVmxTransient->fVectoringDoublePF)
13569	{
13570	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13571	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13572	}
13573	else
13574	{
13575	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13576	Assert(!pVmxTransient->fIsNestedGuest);
13577	hmR0VmxSetPendingXcptDF(pVCpu);
13578	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13579	}
13580	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13581	return VINF_SUCCESS;
13582	}
13583
13584	Assert(!pVmxTransient->fIsNestedGuest);
13585
13586	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13587	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13588	if (pVmxTransient->fVectoringPF)
13589	{
13590	Assert(pVCpu->hm.s.Event.fPending);
13591	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13592	}
13593
13594	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13595	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13596	AssertRCReturn(rc, rc);
13597
13598	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
13599	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
13600
13601	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13602	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13603
13604	Log4Func(("#PF: rc=%Rrc\n", rc));
13605	if (rc == VINF_SUCCESS)
13606	{
13607	/*
13608	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13609	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13610	*/
13611	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13612	TRPMResetTrap(pVCpu);
13613	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13614	return rc;
13615	}
13616
13617	if (rc == VINF_EM_RAW_GUEST_TRAP)
13618	{
13619	if (!pVmxTransient->fVectoringDoublePF)
13620	{
13621	/* It's a guest page fault and needs to be reflected to the guest. */
13622	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
13623	TRPMResetTrap(pVCpu);
13624	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13625	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13626	uGstErrorCode, pVmxTransient->uExitQual);
13627	}
13628	else
13629	{
13630	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13631	TRPMResetTrap(pVCpu);
13632	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13633	hmR0VmxSetPendingXcptDF(pVCpu);
13634	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13635	}
13636
13637	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13638	return VINF_SUCCESS;
13639	}
13640
13641	TRPMResetTrap(pVCpu);
13642	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13643	return rc;
13644	}
13645
13646
13647	/**
13648	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13649	*
13650	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13651	*/
13652	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13653	{
13654	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13655	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13656
13657	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13658	AssertRCReturn(rc, rc);
13659
13660	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13661	{
13662	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13663	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13664
13665	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13666	* provides VM-exit instruction length. If this causes problem later,
13667	* disassemble the instruction like it's done on AMD-V. */
13668	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13669	AssertRCReturn(rc2, rc2);
13670	return rc;
13671	}
13672
13673	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
13674	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13675	return VINF_SUCCESS;
13676	}
13677
13678
13679	/**
13680	* VM-exit exception handler for \#BP (Breakpoint exception).
13681	*
13682	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13683	*/
13684	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13685	{
13686	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13687	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13688
13689	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13690	AssertRCReturn(rc, rc);
13691
13692	if (!pVmxTransient->fIsNestedGuest)
13693	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13694	else
13695	rc = VINF_EM_RAW_GUEST_TRAP;
13696
13697	if (rc == VINF_EM_RAW_GUEST_TRAP)
13698	{
13699	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13700	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13701	rc = VINF_SUCCESS;
13702	}
13703
13704	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
13705	return rc;
13706	}
13707
13708
13709	/**
13710	* VM-exit exception handler for \#AC (Alignment-check exception).
13711	*
13712	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13713	*/
13714	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13715	{
13716	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13717	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13718
13719	/* Re-inject it. We'll detect any nesting before getting here. */
13720	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13721	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13722	return VINF_SUCCESS;
13723	}
13724
13725
13726	/**
13727	* VM-exit exception handler for \#DB (Debug exception).
13728	*
13729	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13730	*/
13731	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13732	{
13733	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13734	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13735
13736	/*
13737	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13738	*/
13739	hmR0VmxReadExitQualVmcs(pVmxTransient);
13740
13741	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13742	uint64_t const uDR6 = X86_DR6_INIT_VAL
13743	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13744	\| X86_DR6_BD \| X86_DR6_BS));
13745
13746	int rc;
13747	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13748	if (!pVmxTransient->fIsNestedGuest)
13749	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13750	else
13751	rc = VINF_EM_RAW_GUEST_TRAP;
13752	Log6Func(("rc=%Rrc\n", rc));
13753	if (rc == VINF_EM_RAW_GUEST_TRAP)
13754	{
13755	/*
13756	* The exception was for the guest. Update DR6, DR7.GD and
13757	* IA32_DEBUGCTL.LBR before forwarding it.
13758	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
13759	*/
13760	VMMRZCallRing3Disable(pVCpu);
13761	HM_DISABLE_PREEMPT(pVCpu);
13762
13763	pCtx->dr[6] &= ~X86_DR6_B_MASK;
13764	pCtx->dr[6] \|= uDR6;
13765	if (CPUMIsGuestDebugStateActive(pVCpu))
13766	ASMSetDR6(pCtx->dr[6]);
13767
13768	HM_RESTORE_PREEMPT();
13769	VMMRZCallRing3Enable(pVCpu);
13770
13771	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
13772	AssertRCReturn(rc, rc);
13773
13774	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13775	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
13776
13777	/* Paranoia. */
13778	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
13779	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
13780
13781	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
13782	AssertRC(rc);
13783
13784	/*
13785	* Raise #DB in the guest.
13786	*
13787	* It is important to reflect exactly what the VM-exit gave us (preserving the
13788	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13789	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13790	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13791	*
13792	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13793	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13794	*/
13795	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13796	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13797	return VINF_SUCCESS;
13798	}
13799
13800	/*
13801	* Not a guest trap, must be a hypervisor related debug event then.
13802	* Update DR6 in case someone is interested in it.
13803	*/
13804	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13805	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13806	CPUMSetHyperDR6(pVCpu, uDR6);
13807
13808	return rc;
13809	}
13810
13811
13812	/**
13813	* Hacks its way around the lovely mesa driver's backdoor accesses.
13814	*
13815	* @sa hmR0SvmHandleMesaDrvGp.
13816	*/
13817	static int hmR0VmxHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13818	{
13819	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
13820	RT_NOREF(pCtx);
13821
13822	/* For now we'll just skip the instruction. */
13823	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13824	}
13825
13826
13827	/**
13828	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
13829	* backdoor logging w/o checking what it is running inside.
13830	*
13831	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
13832	* backdoor port and magic numbers loaded in registers.
13833	*
13834	* @returns true if it is, false if it isn't.
13835	* @sa hmR0SvmIsMesaDrvGp.
13836	*/
13837	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13838	{
13839	/* 0xed: IN eAX,dx */
13840	uint8_t abInstr[1];
13841	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
13842	return false;
13843
13844	/* Check that it is #GP(0). */
13845	if (pVmxTransient->uExitIntErrorCode != 0)
13846	return false;
13847
13848	/* Check magic and port. */
13849	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
13850	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
13851	if (pCtx->rax != UINT32_C(0x564d5868))
13852	return false;
13853	if (pCtx->dx != UINT32_C(0x5658))
13854	return false;
13855
13856	/* Flat ring-3 CS. */
13857	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
13858	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
13859	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
13860	if (pCtx->cs.Attr.n.u2Dpl != 3)
13861	return false;
13862	if (pCtx->cs.u64Base != 0)
13863	return false;
13864
13865	/* Check opcode. */
13866	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
13867	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
13868	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
13869	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
13870	if (RT_FAILURE(rc))
13871	return false;
13872	if (abInstr[0] != 0xed)
13873	return false;
13874
13875	return true;
13876	}
13877
13878
13879	/**
13880	* VM-exit exception handler for \#GP (General-protection exception).
13881	*
13882	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13883	*/
13884	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13885	{
13886	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13887	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13888
13889	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13890	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13891	if (pVmcsInfo->RealMode.fRealOnV86Active)
13892	{ /* likely */ }
13893	else
13894	{
13895	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13896	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
13897	#endif
13898	/*
13899	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
13900	* executing a nested-guest, reflect #GP to the guest or nested-guest.
13901	*/
13902	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13903	AssertRCReturn(rc, rc);
13904	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13905	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13906
13907	if ( pVmxTransient->fIsNestedGuest
13908	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
13909	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
13910	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13911	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13912	else
13913	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
13914	return rc;
13915	}
13916
13917	Assert(CPUMIsGuestInRealModeEx(pCtx));
13918	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13919	Assert(!pVmxTransient->fIsNestedGuest);
13920
13921	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13922	AssertRCReturn(rc, rc);
13923
13924	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13925	if (rcStrict == VINF_SUCCESS)
13926	{
13927	if (!CPUMIsGuestInRealModeEx(pCtx))
13928	{
13929	/*
13930	* The guest is no longer in real-mode, check if we can continue executing the
13931	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13932	*/
13933	pVmcsInfo->RealMode.fRealOnV86Active = false;
13934	if (HMCanExecuteVmxGuest(pVCpu->pVMR0, pVCpu, pCtx))
13935	{
13936	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
13937	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13938	}
13939	else
13940	{
13941	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13942	rcStrict = VINF_EM_RESCHEDULE;
13943	}
13944	}
13945	else
13946	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13947	}
13948	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13949	{
13950	rcStrict = VINF_SUCCESS;
13951	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13952	}
13953	return VBOXSTRICTRC_VAL(rcStrict);
13954	}
13955
13956
13957	/**
13958	* VM-exit exception handler wrapper for all other exceptions that are not handled
13959	* by a specific handler.
13960	*
13961	* This simply re-injects the exception back into the VM without any special
13962	* processing.
13963	*
13964	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13965	*/
13966	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13967	{
13968	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13969
13970	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13971	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13972	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
13973	("uVector=%#x u32XcptBitmap=%#X32\n",
13974	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
13975	NOREF(pVmcsInfo);
13976	#endif
13977
13978	/*
13979	* Re-inject the exception into the guest. This cannot be a double-fault condition which
13980	* would have been handled while checking exits due to event delivery.
13981	*/
13982	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13983
13984	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13985	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13986	AssertRCReturn(rc, rc);
13987	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13988	#endif
13989
13990	#ifdef VBOX_WITH_STATISTICS
13991	switch (uVector)
13992	{
13993	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
13994	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
13995	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
13996	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
13997	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
13998	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
13999	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14000	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14001	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14002	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14003	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14004	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14005	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14006	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14007	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14008	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14009	default:
14010	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14011	break;
14012	}
14013	#endif
14014
14015	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14016	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
14017	NOREF(uVector);
14018
14019	/* Re-inject the original exception into the guest. */
14020	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14021	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14022	return VINF_SUCCESS;
14023	}
14024
14025
14026	/**
14027	* VM-exit exception handler for all exceptions (except NMIs!).
14028	*
14029	* @remarks This may be called for both guests and nested-guests. Take care to not
14030	* make assumptions and avoid doing anything that is not relevant when
14031	* executing a nested-guest (e.g., Mesa driver hacks).
14032	*/
14033	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14034	{
14035	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
14036
14037	/*
14038	* If this VM-exit occurred while delivering an event through the guest IDT, take
14039	* action based on the return code and additional hints (e.g. for page-faults)
14040	* that will be updated in the VMX transient structure.
14041	*/
14042	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14043	if (rcStrict == VINF_SUCCESS)
14044	{
14045	/*
14046	* If an exception caused a VM-exit due to delivery of an event, the original
14047	* event may have to be re-injected into the guest. We shall reinject it and
14048	* continue guest execution. However, page-fault is a complicated case and
14049	* needs additional processing done in hmR0VmxExitXcptPF().
14050	*/
14051	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14052	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14053	if ( !pVCpu->hm.s.Event.fPending
14054	\|\| uVector == X86_XCPT_PF)
14055	{
14056	switch (uVector)
14057	{
14058	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
14059	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
14060	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
14061	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
14062	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
14063	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
14064	default:
14065	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
14066	}
14067	}
14068	/* else: inject pending event before resuming guest execution. */
14069	}
14070	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
14071	{
14072	Assert(pVCpu->hm.s.Event.fPending);
14073	rcStrict = VINF_SUCCESS;
14074	}
14075
14076	return rcStrict;
14077	}
14078	/** @} */
14079
14080
14081	/** @name VM-exit handlers.
14082	* @{
14083	*/
14084	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14085	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14086	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14087
14088	/**
14089	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14090	*/
14091	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14092	{
14093	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14094	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14095	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14096	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14097	return VINF_SUCCESS;
14098	return VINF_EM_RAW_INTERRUPT;
14099	}
14100
14101
14102	/**
14103	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14104	* VM-exit.
14105	*/
14106	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14107	{
14108	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14109	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14110
14111	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14112
14113	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14114	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14115	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14116
14117	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14118	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14119	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14120	NOREF(pVmcsInfo);
14121
14122	VBOXSTRICTRC rcStrict;
14123	switch (uExitIntType)
14124	{
14125	/*
14126	* Host physical NMIs:
14127	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14128	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14129	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14130	*
14131	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14132	* See Intel spec. 27.5.5 "Updating Non-Register State".
14133	*/
14134	case VMX_EXIT_INT_INFO_TYPE_NMI:
14135	{
14136	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14137	break;
14138	}
14139
14140	/*
14141	* Privileged software exceptions (#DB from ICEBP),
14142	* Software exceptions (#BP and #OF),
14143	* Hardware exceptions:
14144	* Process the required exceptions and resume guest execution if possible.
14145	*/
14146	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14147	Assert(uVector == X86_XCPT_DB);
14148	RT_FALL_THRU();
14149	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14150	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14151	RT_FALL_THRU();
14152	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14153	{
14154	NOREF(uVector);
14155	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14156	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14157	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14158	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14159
14160	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14161	break;
14162	}
14163
14164	default:
14165	{
14166	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14167	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14168	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14169	break;
14170	}
14171	}
14172
14173	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14174	return rcStrict;
14175	}
14176
14177
14178	/**
14179	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14180	*/
14181	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14182	{
14183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14184
14185	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14186	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14187	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14188
14189	/* Evaluate and deliver pending events and resume guest execution. */
14190	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14191	return VINF_SUCCESS;
14192	}
14193
14194
14195	/**
14196	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14197	*/
14198	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14199	{
14200	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14201
14202	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14203	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14204	{
14205	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14206	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14207	}
14208
14209	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14210
14211	/*
14212	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14213	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14214	*/
14215	uint32_t fIntrState;
14216	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14217	AssertRC(rc);
14218	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14219	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14220	{
14221	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14222	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14223
14224	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14225	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14226	AssertRC(rc);
14227	}
14228
14229	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14230	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14231
14232	/* Evaluate and deliver pending events and resume guest execution. */
14233	return VINF_SUCCESS;
14234	}
14235
14236
14237	/**
14238	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14239	*/
14240	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14241	{
14242	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14243	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14244	}
14245
14246
14247	/**
14248	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14249	*/
14250	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14251	{
14252	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14253	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14254	}
14255
14256
14257	/**
14258	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14259	*/
14260	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14261	{
14262	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14263
14264	/*
14265	* Get the state we need and update the exit history entry.
14266	*/
14267	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14268	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14269
14270	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14271	AssertRCReturn(rc, rc);
14272
14273	VBOXSTRICTRC rcStrict;
14274	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14275	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14276	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14277	if (!pExitRec)
14278	{
14279	/*
14280	* Regular CPUID instruction execution.
14281	*/
14282	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
14283	if (rcStrict == VINF_SUCCESS)
14284	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14285	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14286	{
14287	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14288	rcStrict = VINF_SUCCESS;
14289	}
14290	}
14291	else
14292	{
14293	/*
14294	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14295	*/
14296	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14297	AssertRCReturn(rc2, rc2);
14298
14299	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14300	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14301
14302	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14303	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14304
14305	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14306	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14307	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14308	}
14309	return rcStrict;
14310	}
14311
14312
14313	/**
14314	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14315	*/
14316	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14317	{
14318	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14319
14320	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14321	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14322	AssertRCReturn(rc, rc);
14323
14324	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14325	return VINF_EM_RAW_EMULATE_INSTR;
14326
14327	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14328	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14329	}
14330
14331
14332	/**
14333	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14334	*/
14335	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14336	{
14337	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14338
14339	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14340	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14341	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14342	AssertRCReturn(rc, rc);
14343
14344	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
14345	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14346	{
14347	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14348	we must reset offsetting on VM-entry. See @bugref{6634}. */
14349	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14350	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14351	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14352	}
14353	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14354	{
14355	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14356	rcStrict = VINF_SUCCESS;
14357	}
14358	return rcStrict;
14359	}
14360
14361
14362	/**
14363	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14364	*/
14365	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14366	{
14367	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14368
14369	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14370	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14371	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14372	AssertRCReturn(rc, rc);
14373
14374	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
14375	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14376	{
14377	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14378	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14379	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14380	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14381	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14382	}
14383	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14384	{
14385	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14386	rcStrict = VINF_SUCCESS;
14387	}
14388	return rcStrict;
14389	}
14390
14391
14392	/**
14393	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14394	*/
14395	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14396	{
14397	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14398
14399	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14400	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14401	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14402	AssertRCReturn(rc, rc);
14403
14404	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14405	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14406	if (RT_LIKELY(rc == VINF_SUCCESS))
14407	{
14408	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14409	Assert(pVmxTransient->cbExitInstr == 2);
14410	}
14411	else
14412	{
14413	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14414	rc = VERR_EM_INTERPRETER;
14415	}
14416	return rc;
14417	}
14418
14419
14420	/**
14421	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14422	*/
14423	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14424	{
14425	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14426
14427	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14428	if (EMAreHypercallInstructionsEnabled(pVCpu))
14429	{
14430	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14431	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14432	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14433	AssertRCReturn(rc, rc);
14434
14435	/* Perform the hypercall. */
14436	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14437	if (rcStrict == VINF_SUCCESS)
14438	{
14439	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14440	AssertRCReturn(rc, rc);
14441	}
14442	else
14443	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14444	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14445	\|\| RT_FAILURE(rcStrict));
14446
14447	/* If the hypercall changes anything other than guest's general-purpose registers,
14448	we would need to reload the guest changed bits here before VM-entry. */
14449	}
14450	else
14451	Log4Func(("Hypercalls not enabled\n"));
14452
14453	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14454	if (RT_FAILURE(rcStrict))
14455	{
14456	hmR0VmxSetPendingXcptUD(pVCpu);
14457	rcStrict = VINF_SUCCESS;
14458	}
14459
14460	return rcStrict;
14461	}
14462
14463
14464	/**
14465	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14466	*/
14467	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14468	{
14469	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14470	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14471
14472	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14473	hmR0VmxReadExitQualVmcs(pVmxTransient);
14474	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14475	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14476	AssertRCReturn(rc, rc);
14477
14478	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
14479
14480	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14481	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14482	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14483	{
14484	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14485	rcStrict = VINF_SUCCESS;
14486	}
14487	else
14488	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14489	VBOXSTRICTRC_VAL(rcStrict)));
14490	return rcStrict;
14491	}
14492
14493
14494	/**
14495	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14496	*/
14497	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14498	{
14499	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14500
14501	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14502	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14503	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14504	AssertRCReturn(rc, rc);
14505
14506	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
14507	if (rcStrict == VINF_SUCCESS)
14508	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14509	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14510	{
14511	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14512	rcStrict = VINF_SUCCESS;
14513	}
14514
14515	return rcStrict;
14516	}
14517
14518
14519	/**
14520	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14521	*/
14522	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14523	{
14524	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14525
14526	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14527	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14528	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14529	AssertRCReturn(rc, rc);
14530
14531	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
14532	if (RT_SUCCESS(rcStrict))
14533	{
14534	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14535	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14536	rcStrict = VINF_SUCCESS;
14537	}
14538
14539	return rcStrict;
14540	}
14541
14542
14543	/**
14544	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14545	* VM-exit.
14546	*/
14547	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14548	{
14549	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14550	return VINF_EM_RESET;
14551	}
14552
14553
14554	/**
14555	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14556	*/
14557	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14558	{
14559	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14560
14561	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14562	AssertRCReturn(rc, rc);
14563
14564	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14565	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14566	rc = VINF_SUCCESS;
14567	else
14568	rc = VINF_EM_HALT;
14569
14570	if (rc != VINF_SUCCESS)
14571	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14572	return rc;
14573	}
14574
14575
14576	/**
14577	* VM-exit handler for instructions that result in a \#UD exception delivered to
14578	* the guest.
14579	*/
14580	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14581	{
14582	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14583	hmR0VmxSetPendingXcptUD(pVCpu);
14584	return VINF_SUCCESS;
14585	}
14586
14587
14588	/**
14589	* VM-exit handler for expiry of the VMX-preemption timer.
14590	*/
14591	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14592	{
14593	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14594
14595	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14596	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14597
14598	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14599	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14600	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14602	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14603	}
14604
14605
14606	/**
14607	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14608	*/
14609	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14610	{
14611	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14612
14613	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14614	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14615	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14616	AssertRCReturn(rc, rc);
14617
14618	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
14619	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14620	: HM_CHANGED_RAISED_XCPT_MASK);
14621
14622	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14623	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14624
14625	return rcStrict;
14626	}
14627
14628
14629	/**
14630	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14631	*/
14632	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14633	{
14634	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14635
14636	#if 1
14637	/** @todo Use VM-exit instruction information. */
14638	return VERR_EM_INTERPRETER;
14639	#else
14640	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14641	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14642	hmR0VmxReadExitQualVmcs(pVmxTransient);
14643	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
14644	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14645	AssertRCReturn(rc, rc);
14646
14647	/* Paranoia. Ensure this has a memory operand. */
14648	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
14649
14650	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
14651	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14652	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
14653	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
14654
14655	RTGCPTR GCPtrDesc;
14656	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
14657
14658	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
14659	GCPtrDesc, uType);
14660	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14661	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14662	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14663	{
14664	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14665	rcStrict = VINF_SUCCESS;
14666	}
14667	return rcStrict;
14668	#endif
14669	}
14670
14671
14672	/**
14673	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14674	* VM-exit.
14675	*/
14676	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14677	{
14678	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14679	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14680	AssertRCReturn(rc, rc);
14681
14682	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14683	if (RT_FAILURE(rc))
14684	return rc;
14685
14686	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14687	NOREF(uInvalidReason);
14688
14689	#ifdef VBOX_STRICT
14690	uint32_t fIntrState;
14691	uint64_t u64Val;
14692	hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14693	hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14694	hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14695
14696	Log4(("uInvalidReason %u\n", uInvalidReason));
14697	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14698	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14699	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14700
14701	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
14702	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14703	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
14704	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
14705	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
14706	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
14707	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
14708	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14709	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
14710	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
14711	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
14712	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14713	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14714	{
14715	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14716	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14717	}
14718	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
14719	#endif
14720
14721	return VERR_VMX_INVALID_GUEST_STATE;
14722	}
14723
14724	/**
14725	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14726	*/
14727	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14728	{
14729	/*
14730	* Cumulative notes of all recognized but unexpected VM-exits.
14731	*
14732	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
14733	* nested-paging is used.
14734	*
14735	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14736	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14737	* this function (and thereby stop VM execution) for handling such instructions.
14738	*
14739	*
14740	* VMX_EXIT_INIT_SIGNAL:
14741	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14742	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14743	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14744	*
14745	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14746	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14747	* See Intel spec. "23.8 Restrictions on VMX operation".
14748	*
14749	* VMX_EXIT_SIPI:
14750	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14751	* activity state is used. We don't make use of it as our guests don't have direct
14752	* access to the host local APIC.
14753	*
14754	* See Intel spec. 25.3 "Other Causes of VM-exits".
14755	*
14756	* VMX_EXIT_IO_SMI:
14757	* VMX_EXIT_SMI:
14758	* This can only happen if we support dual-monitor treatment of SMI, which can be
14759	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14760	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14761	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14762	*
14763	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14764	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14765	*
14766	* VMX_EXIT_ERR_MSR_LOAD:
14767	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14768	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14769	* execution.
14770	*
14771	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14772	*
14773	* VMX_EXIT_ERR_MACHINE_CHECK:
14774	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14775	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14776	* #MC exception abort class exception is raised. We thus cannot assume a
14777	* reasonable chance of continuing any sort of execution and we bail.
14778	*
14779	* See Intel spec. 15.1 "Machine-check Architecture".
14780	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14781	*
14782	* VMX_EXIT_PML_FULL:
14783	* VMX_EXIT_VIRTUALIZED_EOI:
14784	* VMX_EXIT_APIC_WRITE:
14785	* We do not currently support any of these features and thus they are all unexpected
14786	* VM-exits.
14787	*
14788	* VMX_EXIT_GDTR_IDTR_ACCESS:
14789	* VMX_EXIT_LDTR_TR_ACCESS:
14790	* VMX_EXIT_RDRAND:
14791	* VMX_EXIT_RSM:
14792	* VMX_EXIT_VMFUNC:
14793	* VMX_EXIT_ENCLS:
14794	* VMX_EXIT_RDSEED:
14795	* VMX_EXIT_XSAVES:
14796	* VMX_EXIT_XRSTORS:
14797	* VMX_EXIT_UMWAIT:
14798	* VMX_EXIT_TPAUSE:
14799	* These VM-exits are -not- caused unconditionally by execution of the corresponding
14800	* instruction. Any VM-exit for these instructions indicate a hardware problem,
14801	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14802	*
14803	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14804	*/
14805	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14806	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14807	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14808	}
14809
14810
14811	/**
14812	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14813	*/
14814	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14815	{
14816	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14817
14818	/** @todo Optimize this: We currently drag in the whole MSR state
14819	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14820	* MSRs required. That would require changes to IEM and possibly CPUM too.
14821	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14822	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14823	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14824	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14825	switch (idMsr)
14826	{
14827	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14828	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14829	}
14830
14831	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14832	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14833	AssertRCReturn(rc, rc);
14834
14835	Log4Func(("ecx=%#RX32\n", idMsr));
14836
14837	#ifdef VBOX_STRICT
14838	Assert(!pVmxTransient->fIsNestedGuest);
14839	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
14840	{
14841	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14842	&& idMsr != MSR_K6_EFER)
14843	{
14844	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14845	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14846	}
14847	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14848	{
14849	Assert(pVmcsInfo->pvMsrBitmap);
14850	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14851	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14852	{
14853	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14854	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14855	}
14856	}
14857	}
14858	#endif
14859
14860	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
14861	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14862	if (rcStrict == VINF_SUCCESS)
14863	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14864	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14865	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14866	{
14867	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14868	rcStrict = VINF_SUCCESS;
14869	}
14870	else
14871	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14872
14873	return rcStrict;
14874	}
14875
14876
14877	/**
14878	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14879	*/
14880	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14881	{
14882	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14883
14884	/** @todo Optimize this: We currently drag in the whole MSR state
14885	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14886	* MSRs required. That would require changes to IEM and possibly CPUM too.
14887	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14888	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14889	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14890
14891	/*
14892	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14893	* Although we don't need to fetch the base as it will be overwritten shortly, while
14894	* loading guest-state we would also load the entire segment register including limit
14895	* and attributes and thus we need to load them here.
14896	*/
14897	switch (idMsr)
14898	{
14899	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14900	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14901	}
14902
14903	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14904	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14905	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14906	AssertRCReturn(rc, rc);
14907
14908	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14909
14910	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
14911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14912
14913	if (rcStrict == VINF_SUCCESS)
14914	{
14915	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14916
14917	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14918	if ( idMsr == MSR_IA32_APICBASE
14919	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14920	&& idMsr <= MSR_IA32_X2APIC_END))
14921	{
14922	/*
14923	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14924	* When full APIC register virtualization is implemented we'll have to make
14925	* sure APIC state is saved from the VMCS before IEM changes it.
14926	*/
14927	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14928	}
14929	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14930	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14931	else if (idMsr == MSR_K6_EFER)
14932	{
14933	/*
14934	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14935	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14936	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14937	*/
14938	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14939	}
14940
14941	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
14942	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14943	{
14944	switch (idMsr)
14945	{
14946	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14947	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14948	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14949	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14950	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14951	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14952	default:
14953	{
14954	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14955	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14956	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14957	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14958	break;
14959	}
14960	}
14961	}
14962	#ifdef VBOX_STRICT
14963	else
14964	{
14965	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14966	switch (idMsr)
14967	{
14968	case MSR_IA32_SYSENTER_CS:
14969	case MSR_IA32_SYSENTER_EIP:
14970	case MSR_IA32_SYSENTER_ESP:
14971	case MSR_K8_FS_BASE:
14972	case MSR_K8_GS_BASE:
14973	{
14974	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14975	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14976	}
14977
14978	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14979	default:
14980	{
14981	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14982	{
14983	/* EFER MSR writes are always intercepted. */
14984	if (idMsr != MSR_K6_EFER)
14985	{
14986	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14987	idMsr));
14988	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14989	}
14990	}
14991
14992	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14993	{
14994	Assert(pVmcsInfo->pvMsrBitmap);
14995	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14996	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14997	{
14998	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14999	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15000	}
15001	}
15002	break;
15003	}
15004	}
15005	}
15006	#endif /* VBOX_STRICT */
15007	}
15008	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15009	{
15010	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15011	rcStrict = VINF_SUCCESS;
15012	}
15013	else
15014	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15015
15016	return rcStrict;
15017	}
15018
15019
15020	/**
15021	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15022	*/
15023	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15024	{
15025	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15026
15027	/** @todo The guest has likely hit a contended spinlock. We might want to
15028	* poke a schedule different guest VCPU. */
15029	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15030	if (RT_SUCCESS(rc))
15031	return VINF_EM_RAW_INTERRUPT;
15032
15033	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15034	return rc;
15035	}
15036
15037
15038	/**
15039	* VM-exit handler for when the TPR value is lowered below the specified
15040	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15041	*/
15042	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15043	{
15044	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15045	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15046
15047	/*
15048	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15049	* We'll re-evaluate pending interrupts and inject them before the next VM
15050	* entry so we can just continue execution here.
15051	*/
15052	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15053	return VINF_SUCCESS;
15054	}
15055
15056
15057	/**
15058	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15059	* VM-exit.
15060	*
15061	* @retval VINF_SUCCESS when guest execution can continue.
15062	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15063	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15064	* incompatible guest state for VMX execution (real-on-v86 case).
15065	*/
15066	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15067	{
15068	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15069	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15070
15071	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15072	hmR0VmxReadExitQualVmcs(pVmxTransient);
15073	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15074
15075	VBOXSTRICTRC rcStrict;
15076	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15077	uint64_t const uExitQual = pVmxTransient->uExitQual;
15078	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15079	switch (uAccessType)
15080	{
15081	/*
15082	* MOV to CRx.
15083	*/
15084	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15085	{
15086	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15087	AssertRCReturn(rc, rc);
15088
15089	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15090	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15091	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15092	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15093
15094	/*
15095	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15096	* - When nested paging isn't used.
15097	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15098	* - We are executing in the VM debug loop.
15099	*/
15100	Assert( iCrReg != 3
15101	\|\| !pVM->hm.s.fNestedPaging
15102	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15103	\|\| pVCpu->hm.s.fUsingDebugLoop);
15104
15105	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15106	Assert( iCrReg != 8
15107	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15108
15109	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15110	AssertMsg( rcStrict == VINF_SUCCESS
15111	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15112
15113	/*
15114	* This is a kludge for handling switches back to real mode when we try to use
15115	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15116	* deal with special selector values, so we have to return to ring-3 and run
15117	* there till the selector values are V86 mode compatible.
15118	*
15119	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15120	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15121	* this function.
15122	*/
15123	if ( iCrReg == 0
15124	&& rcStrict == VINF_SUCCESS
15125	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15126	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15127	&& (uOldCr0 & X86_CR0_PE)
15128	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15129	{
15130	/** @todo Check selectors rather than returning all the time. */
15131	Assert(!pVmxTransient->fIsNestedGuest);
15132	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15133	rcStrict = VINF_EM_RESCHEDULE_REM;
15134	}
15135	break;
15136	}
15137
15138	/*
15139	* MOV from CRx.
15140	*/
15141	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15142	{
15143	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15144	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15145
15146	/*
15147	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15148	* - When nested paging isn't used.
15149	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15150	* - We are executing in the VM debug loop.
15151	*/
15152	Assert( iCrReg != 3
15153	\|\| !pVM->hm.s.fNestedPaging
15154	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15155	\|\| pVCpu->hm.s.fUsingDebugLoop);
15156
15157	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15158	Assert( iCrReg != 8
15159	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15160
15161	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15162	break;
15163	}
15164
15165	/*
15166	* CLTS (Clear Task-Switch Flag in CR0).
15167	*/
15168	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15169	{
15170	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
15171	break;
15172	}
15173
15174	/*
15175	* LMSW (Load Machine-Status Word into CR0).
15176	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15177	*/
15178	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15179	{
15180	RTGCPTR GCPtrEffDst;
15181	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
15182	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15183	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15184	if (fMemOperand)
15185	{
15186	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
15187	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15188	}
15189	else
15190	GCPtrEffDst = NIL_RTGCPTR;
15191	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15192	break;
15193	}
15194
15195	default:
15196	{
15197	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15198	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15199	}
15200	}
15201
15202	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15203	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15204	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15205
15206	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15207	NOREF(pVM);
15208	return rcStrict;
15209	}
15210
15211
15212	/**
15213	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15214	* VM-exit.
15215	*/
15216	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15217	{
15218	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15219	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15220
15221	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15222	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15223	hmR0VmxReadExitQualVmcs(pVmxTransient);
15224	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15225	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15226	\| CPUMCTX_EXTRN_EFER);
15227	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15228	AssertRCReturn(rc, rc);
15229
15230	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15231	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15232	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15233	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15234	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15235	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15236	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15237	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15238
15239	/*
15240	* Update exit history to see if this exit can be optimized.
15241	*/
15242	VBOXSTRICTRC rcStrict;
15243	PCEMEXITREC pExitRec = NULL;
15244	if ( !fGstStepping
15245	&& !fDbgStepping)
15246	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15247	!fIOString
15248	? !fIOWrite
15249	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15250	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15251	: !fIOWrite
15252	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15253	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15254	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15255	if (!pExitRec)
15256	{
15257	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15258	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15259
15260	uint32_t const cbValue = s_aIOSizes[uIOSize];
15261	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
15262	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15263	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15264	if (fIOString)
15265	{
15266	/*
15267	* INS/OUTS - I/O String instruction.
15268	*
15269	* Use instruction-information if available, otherwise fall back on
15270	* interpreting the instruction.
15271	*/
15272	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15273	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15274	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15275	if (fInsOutsInfo)
15276	{
15277	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15278	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15279	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15280	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15281	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15282	if (fIOWrite)
15283	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15284	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15285	else
15286	{
15287	/*
15288	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15289	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15290	* See Intel Instruction spec. for "INS".
15291	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15292	*/
15293	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15294	}
15295	}
15296	else
15297	rcStrict = IEMExecOne(pVCpu);
15298
15299	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15300	fUpdateRipAlready = true;
15301	}
15302	else
15303	{
15304	/*
15305	* IN/OUT - I/O instruction.
15306	*/
15307	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15308	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15309	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15310	if (fIOWrite)
15311	{
15312	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15313	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15314	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15315	&& !pCtx->eflags.Bits.u1TF)
15316	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15317	}
15318	else
15319	{
15320	uint32_t u32Result = 0;
15321	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15322	if (IOM_SUCCESS(rcStrict))
15323	{
15324	/* Save result of I/O IN instr. in AL/AX/EAX. */
15325	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15326	}
15327	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15328	&& !pCtx->eflags.Bits.u1TF)
15329	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15330	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15331	}
15332	}
15333
15334	if (IOM_SUCCESS(rcStrict))
15335	{
15336	if (!fUpdateRipAlready)
15337	{
15338	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15339	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15340	}
15341
15342	/*
15343	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15344	* while booting Fedora 17 64-bit guest.
15345	*
15346	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15347	*/
15348	if (fIOString)
15349	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15350
15351	/*
15352	* If any I/O breakpoints are armed, we need to check if one triggered
15353	* and take appropriate action.
15354	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15355	*/
15356	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15357	AssertRCReturn(rc, rc);
15358
15359	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15360	* execution engines about whether hyper BPs and such are pending. */
15361	uint32_t const uDr7 = pCtx->dr[7];
15362	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15363	&& X86_DR7_ANY_RW_IO(uDr7)
15364	&& (pCtx->cr4 & X86_CR4_DE))
15365	\|\| DBGFBpIsHwIoArmed(pVM)))
15366	{
15367	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15368
15369	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15370	VMMRZCallRing3Disable(pVCpu);
15371	HM_DISABLE_PREEMPT(pVCpu);
15372
15373	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15374
15375	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15376	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15377	{
15378	/* Raise #DB. */
15379	if (fIsGuestDbgActive)
15380	ASMSetDR6(pCtx->dr[6]);
15381	if (pCtx->dr[7] != uDr7)
15382	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15383
15384	hmR0VmxSetPendingXcptDB(pVCpu);
15385	}
15386	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15387	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15388	else if ( rcStrict2 != VINF_SUCCESS
15389	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15390	rcStrict = rcStrict2;
15391	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15392
15393	HM_RESTORE_PREEMPT();
15394	VMMRZCallRing3Enable(pVCpu);
15395	}
15396	}
15397
15398	#ifdef VBOX_STRICT
15399	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15400	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15401	Assert(!fIOWrite);
15402	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15403	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15404	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15405	Assert(fIOWrite);
15406	else
15407	{
15408	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15409	* statuses, that the VMM device and some others may return. See
15410	* IOM_SUCCESS() for guidance. */
15411	AssertMsg( RT_FAILURE(rcStrict)
15412	\|\| rcStrict == VINF_SUCCESS
15413	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15414	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15415	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15416	\|\| rcStrict == VINF_EM_RAW_TO_R3
15417	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15418	# endif
15419	}
15420	#endif
15421	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15422	}
15423	else
15424	{
15425	/*
15426	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15427	*/
15428	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15429	AssertRCReturn(rc2, rc2);
15430	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15431	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15432	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15433	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15434	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15435	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15436
15437	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15438	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15439
15440	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15441	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15442	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15443	}
15444	return rcStrict;
15445	}
15446
15447
15448	/**
15449	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15450	* VM-exit.
15451	*/
15452	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15453	{
15454	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15455
15456	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15457	hmR0VmxReadExitQualVmcs(pVmxTransient);
15458	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15459	{
15460	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15461	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15462	{
15463	uint32_t uErrCode;
15464	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15465	{
15466	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15467	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15468	}
15469	else
15470	uErrCode = 0;
15471
15472	RTGCUINTPTR GCPtrFaultAddress;
15473	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15474	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15475	else
15476	GCPtrFaultAddress = 0;
15477
15478	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15479
15480	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15481	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
15482
15483	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15484	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15486	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15487	}
15488	}
15489
15490	/* Fall back to the interpreter to emulate the task-switch. */
15491	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15492	return VERR_EM_INTERPRETER;
15493	}
15494
15495
15496	/**
15497	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15498	*/
15499	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15500	{
15501	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15502
15503	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15504	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15505	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15506	AssertRC(rc);
15507	return VINF_EM_DBG_STEPPED;
15508	}
15509
15510
15511	/**
15512	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15513	*/
15514	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15515	{
15516	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15517	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15518
15519	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15520	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15521	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15522	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15523	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15524
15525	/*
15526	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15527	*/
15528	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15529	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15530	{
15531	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15532	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15533	{
15534	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15535	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15536	}
15537	}
15538	else
15539	{
15540	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15541	return rcStrict;
15542	}
15543
15544	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15545	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15546	hmR0VmxReadExitQualVmcs(pVmxTransient);
15547	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15548	AssertRCReturn(rc, rc);
15549
15550	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15551	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15552	switch (uAccessType)
15553	{
15554	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15555	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15556	{
15557	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15558	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15559	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15560
15561	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15562	GCPhys &= PAGE_BASE_GC_MASK;
15563	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15564	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15565	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15566
15567	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15568	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15569	rcStrict = IOMMMIOPhysHandler(pVM, pVCpu,
15570	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15571	CPUMCTX2CORE(pCtx), GCPhys);
15572	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15573	if ( rcStrict == VINF_SUCCESS
15574	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15575	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15576	{
15577	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15578	\| HM_CHANGED_GUEST_APIC_TPR);
15579	rcStrict = VINF_SUCCESS;
15580	}
15581	break;
15582	}
15583
15584	default:
15585	{
15586	Log4Func(("uAccessType=%#x\n", uAccessType));
15587	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15588	break;
15589	}
15590	}
15591
15592	if (rcStrict != VINF_SUCCESS)
15593	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15594	return rcStrict;
15595	}
15596
15597
15598	/**
15599	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15600	* VM-exit.
15601	*/
15602	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15603	{
15604	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15605	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15606
15607	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15608	if (!pVmxTransient->fIsNestedGuest)
15609	{
15610	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15611	if (pVmxTransient->fWasGuestDebugStateActive)
15612	{
15613	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15614	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15615	}
15616
15617	if ( !pVCpu->hm.s.fSingleInstruction
15618	&& !pVmxTransient->fWasHyperDebugStateActive)
15619	{
15620	Assert(!DBGFIsStepping(pVCpu));
15621	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15622
15623	/* Don't intercept MOV DRx any more. */
15624	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15625	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15626	AssertRC(rc);
15627
15628	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15629	VMMRZCallRing3Disable(pVCpu);
15630	HM_DISABLE_PREEMPT(pVCpu);
15631
15632	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15633	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15634	Assert(CPUMIsGuestDebugStateActive(pVCpu));
15635
15636	HM_RESTORE_PREEMPT();
15637	VMMRZCallRing3Enable(pVCpu);
15638
15639	#ifdef VBOX_WITH_STATISTICS
15640	hmR0VmxReadExitQualVmcs(pVmxTransient);
15641	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15642	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15643	else
15644	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15645	#endif
15646	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15647	return VINF_SUCCESS;
15648	}
15649	}
15650
15651	/*
15652	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15653	* The EFER MSR is always up-to-date.
15654	* Update the segment registers and DR7 from the CPU.
15655	*/
15656	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15657	hmR0VmxReadExitQualVmcs(pVmxTransient);
15658	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15659	AssertRCReturn(rc, rc);
15660	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15661
15662	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15663	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15664	{
15665	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15666	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15667	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15668	if (RT_SUCCESS(rc))
15669	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15670	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15671	}
15672	else
15673	{
15674	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15675	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15676	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15677	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15678	}
15679
15680	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15681	if (RT_SUCCESS(rc))
15682	{
15683	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15684	AssertRCReturn(rc2, rc2);
15685	return VINF_SUCCESS;
15686	}
15687	return rc;
15688	}
15689
15690
15691	/**
15692	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15693	* Conditional VM-exit.
15694	*/
15695	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15696	{
15697	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15698	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15699
15700	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15701	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15702	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15703	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15704	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15705
15706	/*
15707	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15708	*/
15709	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15710	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15711	{
15712	/*
15713	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15714	* instruction emulation to inject the original event. Otherwise, injecting the original event
15715	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
15716	*/
15717	if (!pVCpu->hm.s.Event.fPending)
15718	{ /* likely */ }
15719	else
15720	{
15721	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15722	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15723	/** @todo NSTVMX: Think about how this should be handled. */
15724	if (pVmxTransient->fIsNestedGuest)
15725	return VERR_VMX_IPE_3;
15726	#endif
15727	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15728	}
15729	}
15730	else
15731	{
15732	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15733	return rcStrict;
15734	}
15735
15736	/*
15737	* Get sufficient state and update the exit history entry.
15738	*/
15739	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15740	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15741	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15742	AssertRCReturn(rc, rc);
15743
15744	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15745	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15746	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15747	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15748	if (!pExitRec)
15749	{
15750	/*
15751	* If we succeed, resume guest execution.
15752	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15753	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15754	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15755	* weird case. See @bugref{6043}.
15756	*/
15757	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15758	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15759	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15760	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15761	if ( rcStrict == VINF_SUCCESS
15762	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15763	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15764	{
15765	/* Successfully handled MMIO operation. */
15766	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15767	\| HM_CHANGED_GUEST_APIC_TPR);
15768	rcStrict = VINF_SUCCESS;
15769	}
15770	}
15771	else
15772	{
15773	/*
15774	* Frequent exit or something needing probing. Call EMHistoryExec.
15775	*/
15776	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15777	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15778
15779	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15780	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15781
15782	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15783	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15784	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15785	}
15786	return rcStrict;
15787	}
15788
15789
15790	/**
15791	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15792	* VM-exit.
15793	*/
15794	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15795	{
15796	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15797	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15798
15799	hmR0VmxReadExitQualVmcs(pVmxTransient);
15800	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15801	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15802	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15803	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15804	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15805
15806	/*
15807	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15808	*/
15809	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15810	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15811	{
15812	/*
15813	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
15814	* we shall resolve the nested #PF and re-inject the original event.
15815	*/
15816	if (pVCpu->hm.s.Event.fPending)
15817	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
15818	}
15819	else
15820	{
15821	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15822	return rcStrict;
15823	}
15824
15825	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15826	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15827	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15828	AssertRCReturn(rc, rc);
15829
15830	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15831	uint64_t const uExitQual = pVmxTransient->uExitQual;
15832	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
15833
15834	RTGCUINT uErrorCode = 0;
15835	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15836	uErrorCode \|= X86_TRAP_PF_ID;
15837	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15838	uErrorCode \|= X86_TRAP_PF_RW;
15839	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15840	uErrorCode \|= X86_TRAP_PF_P;
15841
15842	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15843	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15844	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
15845
15846	/*
15847	* Handle the pagefault trap for the nested shadow table.
15848	*/
15849	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15850	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15851	TRPMResetTrap(pVCpu);
15852
15853	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15854	if ( rcStrict == VINF_SUCCESS
15855	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15856	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15857	{
15858	/* Successfully synced our nested page tables. */
15859	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15860	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15861	return VINF_SUCCESS;
15862	}
15863
15864	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15865	return rcStrict;
15866	}
15867
15868
15869	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15870	/**
15871	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15872	*/
15873	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15874	{
15875	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15876
15877	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15878	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15879	hmR0VmxReadExitQualVmcs(pVmxTransient);
15880	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15881	\| CPUMCTX_EXTRN_HWVIRT
15882	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15883	AssertRCReturn(rc, rc);
15884
15885	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15886
15887	VMXVEXITINFO ExitInfo;
15888	RT_ZERO(ExitInfo);
15889	ExitInfo.uReason = pVmxTransient->uExitReason;
15890	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15891	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15892	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15893	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15894
15895	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15896	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15897	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15898	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15899	{
15900	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15901	rcStrict = VINF_SUCCESS;
15902	}
15903	return rcStrict;
15904	}
15905
15906
15907	/**
15908	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15909	*/
15910	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15911	{
15912	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15913
15914	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15915	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15916	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15917	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15918	AssertRCReturn(rc, rc);
15919
15920	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15921
15922	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
15923	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
15924	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
15925	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15926	{
15927	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15928	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
15929	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15930	}
15931	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15932	return rcStrict;
15933	}
15934
15935
15936	/**
15937	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15938	*/
15939	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15940	{
15941	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15942
15943	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15944	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15945	hmR0VmxReadExitQualVmcs(pVmxTransient);
15946	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15947	\| CPUMCTX_EXTRN_HWVIRT
15948	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15949	AssertRCReturn(rc, rc);
15950
15951	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15952
15953	VMXVEXITINFO ExitInfo;
15954	RT_ZERO(ExitInfo);
15955	ExitInfo.uReason = pVmxTransient->uExitReason;
15956	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15957	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15958	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15959	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15960
15961	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15962	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15963	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15964	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15965	{
15966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15967	rcStrict = VINF_SUCCESS;
15968	}
15969	return rcStrict;
15970	}
15971
15972
15973	/**
15974	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15975	*/
15976	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15977	{
15978	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15979
15980	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15981	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15982	hmR0VmxReadExitQualVmcs(pVmxTransient);
15983	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15984	\| CPUMCTX_EXTRN_HWVIRT
15985	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15986	AssertRCReturn(rc, rc);
15987
15988	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15989
15990	VMXVEXITINFO ExitInfo;
15991	RT_ZERO(ExitInfo);
15992	ExitInfo.uReason = pVmxTransient->uExitReason;
15993	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15994	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15995	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15996	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15997
15998	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15999	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16000	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16001	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16002	{
16003	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16004	rcStrict = VINF_SUCCESS;
16005	}
16006	return rcStrict;
16007	}
16008
16009
16010	/**
16011	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16012	*/
16013	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16014	{
16015	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16016
16017	/*
16018	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16019	* thus might not need to import the shadow VMCS state, it's safer just in case
16020	* code elsewhere dares look at unsynced VMCS fields.
16021	*/
16022	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16023	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16024	hmR0VmxReadExitQualVmcs(pVmxTransient);
16025	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16026	\| CPUMCTX_EXTRN_HWVIRT
16027	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16028	AssertRCReturn(rc, rc);
16029
16030	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16031
16032	VMXVEXITINFO ExitInfo;
16033	RT_ZERO(ExitInfo);
16034	ExitInfo.uReason = pVmxTransient->uExitReason;
16035	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16036	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16037	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16038	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16039	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16040
16041	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16042	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16043	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16044	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16045	{
16046	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16047	rcStrict = VINF_SUCCESS;
16048	}
16049	return rcStrict;
16050	}
16051
16052
16053	/**
16054	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16055	*/
16056	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16057	{
16058	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16059
16060	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16061	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16062	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16063	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16064	AssertRCReturn(rc, rc);
16065
16066	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16067
16068	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16069	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
16070	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16071	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16072	{
16073	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16074	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16075	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16076	}
16077	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16078	return rcStrict;
16079	}
16080
16081
16082	/**
16083	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16084	*/
16085	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16086	{
16087	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16088
16089	/*
16090	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16091	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16092	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16093	*/
16094	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16095	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16096	hmR0VmxReadExitQualVmcs(pVmxTransient);
16097	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16098	\| CPUMCTX_EXTRN_HWVIRT
16099	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16100	AssertRCReturn(rc, rc);
16101
16102	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16103
16104	VMXVEXITINFO ExitInfo;
16105	RT_ZERO(ExitInfo);
16106	ExitInfo.uReason = pVmxTransient->uExitReason;
16107	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16108	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16109	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16110	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16111	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16112
16113	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16114	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16115	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16116	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16117	{
16118	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16119	rcStrict = VINF_SUCCESS;
16120	}
16121	return rcStrict;
16122	}
16123
16124
16125	/**
16126	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16127	*/
16128	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16129	{
16130	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16131
16132	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16133	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16134	\| CPUMCTX_EXTRN_HWVIRT
16135	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16136	AssertRCReturn(rc, rc);
16137
16138	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16139
16140	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
16141	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16142	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16143	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16144	{
16145	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16146	rcStrict = VINF_SUCCESS;
16147	}
16148	return rcStrict;
16149	}
16150
16151
16152	/**
16153	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16154	*/
16155	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16156	{
16157	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16158
16159	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16160	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16161	hmR0VmxReadExitQualVmcs(pVmxTransient);
16162	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16163	\| CPUMCTX_EXTRN_HWVIRT
16164	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16165	AssertRCReturn(rc, rc);
16166
16167	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16168
16169	VMXVEXITINFO ExitInfo;
16170	RT_ZERO(ExitInfo);
16171	ExitInfo.uReason = pVmxTransient->uExitReason;
16172	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16173	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16174	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16175	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16176
16177	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16178	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16179	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16180	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16181	{
16182	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16183	rcStrict = VINF_SUCCESS;
16184	}
16185	return rcStrict;
16186	}
16187
16188
16189	/**
16190	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16191	*/
16192	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16193	{
16194	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16195
16196	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16197	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16198	hmR0VmxReadExitQualVmcs(pVmxTransient);
16199	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16200	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16201	AssertRCReturn(rc, rc);
16202
16203	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16204
16205	VMXVEXITINFO ExitInfo;
16206	RT_ZERO(ExitInfo);
16207	ExitInfo.uReason = pVmxTransient->uExitReason;
16208	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16209	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16210	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16211	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16212
16213	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16214	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16215	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16216	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16217	{
16218	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16219	rcStrict = VINF_SUCCESS;
16220	}
16221	return rcStrict;
16222	}
16223	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16224	/** @} */
16225
16226
16227	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16228	/** @name Nested-guest VM-exit handlers.
16229	* @{
16230	*/
16231	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16232	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16233	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16234
16235	/**
16236	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16237	* Conditional VM-exit.
16238	*/
16239	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16240	{
16241	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16242
16243	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16244
16245	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16246	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16247	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16248
16249	switch (uExitIntType)
16250	{
16251	/*
16252	* Physical NMIs:
16253	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16254	*/
16255	case VMX_EXIT_INT_INFO_TYPE_NMI:
16256	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16257
16258	/*
16259	* Hardware exceptions,
16260	* Software exceptions,
16261	* Privileged software exceptions:
16262	* Figure out if the exception must be delivered to the guest or the nested-guest.
16263	*/
16264	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16265	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16266	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16267	{
16268	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16269	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16270	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16271	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16272
16273	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16274	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16275	pVmxTransient->uExitIntErrorCode);
16276	if (fIntercept)
16277	{
16278	/* Exit qualification is required for debug and page-fault exceptions. */
16279	hmR0VmxReadExitQualVmcs(pVmxTransient);
16280
16281	/*
16282	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16283	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16284	* length. However, if delivery of a software interrupt, software exception or privileged
16285	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16286	*/
16287	VMXVEXITINFO ExitInfo;
16288	RT_ZERO(ExitInfo);
16289	ExitInfo.uReason = pVmxTransient->uExitReason;
16290	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16291	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16292
16293	VMXVEXITEVENTINFO ExitEventInfo;
16294	RT_ZERO(ExitEventInfo);
16295	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16296	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16297	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16298	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16299
16300	#ifdef DEBUG_ramshankar
16301	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16302	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
16303	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16304	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16305	{
16306	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
16307	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
16308	}
16309	#endif
16310	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16311	}
16312
16313	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16314	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16315	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16316	}
16317
16318	/*
16319	* Software interrupts:
16320	* VM-exits cannot be caused by software interrupts.
16321	*
16322	* External interrupts:
16323	* This should only happen when "acknowledge external interrupts on VM-exit"
16324	* control is set. However, we never set this when executing a guest or
16325	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16326	* the guest.
16327	*/
16328	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16329	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16330	default:
16331	{
16332	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16333	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16334	}
16335	}
16336	}
16337
16338
16339	/**
16340	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16341	* Unconditional VM-exit.
16342	*/
16343	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16344	{
16345	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16346	return IEMExecVmxVmexitTripleFault(pVCpu);
16347	}
16348
16349
16350	/**
16351	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16352	*/
16353	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16354	{
16355	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16356
16357	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16358	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16359	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16360	}
16361
16362
16363	/**
16364	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16365	*/
16366	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16367	{
16368	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16369
16370	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16371	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16372	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16373	}
16374
16375
16376	/**
16377	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16378	* Unconditional VM-exit.
16379	*/
16380	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16381	{
16382	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16383
16384	hmR0VmxReadExitQualVmcs(pVmxTransient);
16385	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16386	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16387	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16388
16389	VMXVEXITINFO ExitInfo;
16390	RT_ZERO(ExitInfo);
16391	ExitInfo.uReason = pVmxTransient->uExitReason;
16392	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16393	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16394
16395	VMXVEXITEVENTINFO ExitEventInfo;
16396	RT_ZERO(ExitEventInfo);
16397	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16398	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16399	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16400	}
16401
16402
16403	/**
16404	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16405	*/
16406	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16407	{
16408	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16409
16410	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16411	{
16412	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16413	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16414	}
16415	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16416	}
16417
16418
16419	/**
16420	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16421	*/
16422	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16423	{
16424	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16425
16426	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16427	{
16428	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16429	hmR0VmxReadExitQualVmcs(pVmxTransient);
16430
16431	VMXVEXITINFO ExitInfo;
16432	RT_ZERO(ExitInfo);
16433	ExitInfo.uReason = pVmxTransient->uExitReason;
16434	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16435	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16436	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16437	}
16438	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16439	}
16440
16441
16442	/**
16443	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16444	*/
16445	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16446	{
16447	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16448
16449	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16450	{
16451	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16452	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16453	}
16454	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16455	}
16456
16457
16458	/**
16459	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16460	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16461	*/
16462	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16463	{
16464	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16465
16466	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16467	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16468
16469	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16470
16471	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16472	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16473	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16474
16475	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16476	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16477	u64VmcsField &= UINT64_C(0xffffffff);
16478
16479	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16480	{
16481	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16482	hmR0VmxReadExitQualVmcs(pVmxTransient);
16483
16484	VMXVEXITINFO ExitInfo;
16485	RT_ZERO(ExitInfo);
16486	ExitInfo.uReason = pVmxTransient->uExitReason;
16487	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16488	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16489	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16490	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16491	}
16492
16493	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16494	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16495	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16496	}
16497
16498
16499	/**
16500	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16501	*/
16502	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16503	{
16504	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16505
16506	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16507	{
16508	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16509	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16510	}
16511
16512	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16513	}
16514
16515
16516	/**
16517	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16518	* Conditional VM-exit.
16519	*/
16520	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16521	{
16522	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16523
16524	hmR0VmxReadExitQualVmcs(pVmxTransient);
16525	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16526
16527	VBOXSTRICTRC rcStrict;
16528	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16529	switch (uAccessType)
16530	{
16531	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16532	{
16533	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16534	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16535	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16536	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16537
16538	bool fIntercept;
16539	switch (iCrReg)
16540	{
16541	case 0:
16542	case 4:
16543	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16544	break;
16545
16546	case 3:
16547	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16548	break;
16549
16550	case 8:
16551	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16552	break;
16553
16554	default:
16555	fIntercept = false;
16556	break;
16557	}
16558	if (fIntercept)
16559	{
16560	VMXVEXITINFO ExitInfo;
16561	RT_ZERO(ExitInfo);
16562	ExitInfo.uReason = pVmxTransient->uExitReason;
16563	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16564	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16565	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16566	}
16567	else
16568	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16569	break;
16570	}
16571
16572	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16573	{
16574	/*
16575	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16576	* CR2 reads do not cause a VM-exit.
16577	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16578	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16579	*/
16580	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16581	if ( iCrReg == 3
16582	\|\| iCrReg == 8)
16583	{
16584	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16585	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16586	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16587	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16588	{
16589	VMXVEXITINFO ExitInfo;
16590	RT_ZERO(ExitInfo);
16591	ExitInfo.uReason = pVmxTransient->uExitReason;
16592	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16593	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16594	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16595	}
16596	else
16597	{
16598	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16599	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16600	}
16601	}
16602	else
16603	{
16604	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16605	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16606	}
16607	break;
16608	}
16609
16610	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16611	{
16612	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16613	Assert(pVmcsNstGst);
16614	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16615	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16616	if ( (uGstHostMask & X86_CR0_TS)
16617	&& (uReadShadow & X86_CR0_TS))
16618	{
16619	VMXVEXITINFO ExitInfo;
16620	RT_ZERO(ExitInfo);
16621	ExitInfo.uReason = pVmxTransient->uExitReason;
16622	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16623	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16624	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16625	}
16626	else
16627	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
16628	break;
16629	}
16630
16631	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16632	{
16633	RTGCPTR GCPtrEffDst;
16634	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16635	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16636	if (fMemOperand)
16637	{
16638	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16639	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16640	}
16641	else
16642	GCPtrEffDst = NIL_RTGCPTR;
16643
16644	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16645	{
16646	VMXVEXITINFO ExitInfo;
16647	RT_ZERO(ExitInfo);
16648	ExitInfo.uReason = pVmxTransient->uExitReason;
16649	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16650	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16651	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16652	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16653	}
16654	else
16655	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
16656	break;
16657	}
16658
16659	default:
16660	{
16661	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16662	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16663	}
16664	}
16665
16666	if (rcStrict == VINF_IEM_RAISED_XCPT)
16667	{
16668	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16669	rcStrict = VINF_SUCCESS;
16670	}
16671	return rcStrict;
16672	}
16673
16674
16675	/**
16676	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16677	* Conditional VM-exit.
16678	*/
16679	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16680	{
16681	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16682
16683	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16684	{
16685	hmR0VmxReadExitQualVmcs(pVmxTransient);
16686	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16687
16688	VMXVEXITINFO ExitInfo;
16689	RT_ZERO(ExitInfo);
16690	ExitInfo.uReason = pVmxTransient->uExitReason;
16691	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16692	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16693	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16694	}
16695	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16696	}
16697
16698
16699	/**
16700	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16701	* Conditional VM-exit.
16702	*/
16703	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16704	{
16705	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16706
16707	hmR0VmxReadExitQualVmcs(pVmxTransient);
16708
16709	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16710	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16711	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16712
16713	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16714	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16715	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16716	{
16717	/*
16718	* IN/OUT instruction:
16719	* - Provides VM-exit instruction length.
16720	*
16721	* INS/OUTS instruction:
16722	* - Provides VM-exit instruction length.
16723	* - Provides Guest-linear address.
16724	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16725	*/
16726	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16727	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16728
16729	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16730	pVmxTransient->ExitInstrInfo.u = 0;
16731	pVmxTransient->uGuestLinearAddr = 0;
16732
16733	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16734	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16735	if (fIOString)
16736	{
16737	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16738	if (fVmxInsOutsInfo)
16739	{
16740	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16741	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16742	}
16743	}
16744
16745	VMXVEXITINFO ExitInfo;
16746	RT_ZERO(ExitInfo);
16747	ExitInfo.uReason = pVmxTransient->uExitReason;
16748	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16749	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16750	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16751	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16752	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16753	}
16754	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16755	}
16756
16757
16758	/**
16759	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16760	*/
16761	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16762	{
16763	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16764
16765	uint32_t fMsrpm;
16766	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16767	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16768	else
16769	fMsrpm = VMXMSRPM_EXIT_RD;
16770
16771	if (fMsrpm & VMXMSRPM_EXIT_RD)
16772	{
16773	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16774	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16775	}
16776	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16777	}
16778
16779
16780	/**
16781	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16782	*/
16783	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16784	{
16785	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16786
16787	uint32_t fMsrpm;
16788	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16789	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16790	else
16791	fMsrpm = VMXMSRPM_EXIT_WR;
16792
16793	if (fMsrpm & VMXMSRPM_EXIT_WR)
16794	{
16795	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16796	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16797	}
16798	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16799	}
16800
16801
16802	/**
16803	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16804	*/
16805	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16806	{
16807	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16808
16809	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16810	{
16811	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16812	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16813	}
16814	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16815	}
16816
16817
16818	/**
16819	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16820	* VM-exit.
16821	*/
16822	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16823	{
16824	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16825
16826	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16827	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16828	VMXVEXITINFO ExitInfo;
16829	RT_ZERO(ExitInfo);
16830	ExitInfo.uReason = pVmxTransient->uExitReason;
16831	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16832	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16833	}
16834
16835
16836	/**
16837	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16838	*/
16839	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16840	{
16841	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16842
16843	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16844	{
16845	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16846	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16847	}
16848	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16849	}
16850
16851
16852	/**
16853	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16854	*/
16855	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16856	{
16857	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16858
16859	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16860	* PAUSE when executing a nested-guest? If it does not, we would not need
16861	* to check for the intercepts here. Just call VM-exit... */
16862
16863	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16864	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16865	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16866	{
16867	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16868	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16869	}
16870	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16871	}
16872
16873
16874	/**
16875	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16876	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16877	*/
16878	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16879	{
16880	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16881
16882	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16883	{
16884	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16885	VMXVEXITINFO ExitInfo;
16886	RT_ZERO(ExitInfo);
16887	ExitInfo.uReason = pVmxTransient->uExitReason;
16888	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16889	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16890	}
16891	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16892	}
16893
16894
16895	/**
16896	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16897	* VM-exit.
16898	*/
16899	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16900	{
16901	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16902
16903	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16904	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16905	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16906	hmR0VmxReadExitQualVmcs(pVmxTransient);
16907
16908	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16909
16910	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
16911	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
16912
16913	VMXVEXITINFO ExitInfo;
16914	RT_ZERO(ExitInfo);
16915	ExitInfo.uReason = pVmxTransient->uExitReason;
16916	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16917	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16918
16919	VMXVEXITEVENTINFO ExitEventInfo;
16920	RT_ZERO(ExitEventInfo);
16921	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16922	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16923	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
16924	}
16925
16926
16927	/**
16928	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
16929	* Conditional VM-exit.
16930	*/
16931	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16932	{
16933	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16934
16935	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
16936	hmR0VmxReadExitQualVmcs(pVmxTransient);
16937	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16938	}
16939
16940
16941	/**
16942	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
16943	* Conditional VM-exit.
16944	*/
16945	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16946	{
16947	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16948
16949	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
16950	hmR0VmxReadExitQualVmcs(pVmxTransient);
16951	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16952	}
16953
16954
16955	/**
16956	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16957	*/
16958	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16959	{
16960	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16961
16962	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16963	{
16964	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16965	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16966	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16967	}
16968	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16969	}
16970
16971
16972	/**
16973	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16974	*/
16975	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16976	{
16977	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16978
16979	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16980	{
16981	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16982	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16983	}
16984	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16985	}
16986
16987
16988	/**
16989	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16990	*/
16991	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16992	{
16993	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16994
16995	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16996	{
16997	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
16998	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16999	hmR0VmxReadExitQualVmcs(pVmxTransient);
17000	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17001
17002	VMXVEXITINFO ExitInfo;
17003	RT_ZERO(ExitInfo);
17004	ExitInfo.uReason = pVmxTransient->uExitReason;
17005	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17006	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17007	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17008	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17009	}
17010	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17011	}
17012
17013
17014	/**
17015	* Nested-guest VM-exit handler for invalid-guest state
17016	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17017	*/
17018	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17019	{
17020	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17021
17022	/*
17023	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17024	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17025	* Handle it like it's in an invalid guest state of the outer guest.
17026	*
17027	* When the fast path is implemented, this should be changed to cause the corresponding
17028	* nested-guest VM-exit.
17029	*/
17030	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17031	}
17032
17033
17034	/**
17035	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17036	* and only provide the instruction length.
17037	*
17038	* Unconditional VM-exit.
17039	*/
17040	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17041	{
17042	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17043
17044	#ifdef VBOX_STRICT
17045	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17046	switch (pVmxTransient->uExitReason)
17047	{
17048	case VMX_EXIT_ENCLS:
17049	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17050	break;
17051
17052	case VMX_EXIT_VMFUNC:
17053	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17054	break;
17055	}
17056	#endif
17057
17058	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17059	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17060	}
17061
17062
17063	/**
17064	* Nested-guest VM-exit handler for instructions that provide instruction length as
17065	* well as more information.
17066	*
17067	* Unconditional VM-exit.
17068	*/
17069	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17070	{
17071	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17072
17073	#ifdef VBOX_STRICT
17074	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17075	switch (pVmxTransient->uExitReason)
17076	{
17077	case VMX_EXIT_GDTR_IDTR_ACCESS:
17078	case VMX_EXIT_LDTR_TR_ACCESS:
17079	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17080	break;
17081
17082	case VMX_EXIT_RDRAND:
17083	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17084	break;
17085
17086	case VMX_EXIT_RDSEED:
17087	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17088	break;
17089
17090	case VMX_EXIT_XSAVES:
17091	case VMX_EXIT_XRSTORS:
17092	/** @todo NSTVMX: Verify XSS-bitmap. */
17093	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17094	break;
17095
17096	case VMX_EXIT_UMWAIT:
17097	case VMX_EXIT_TPAUSE:
17098	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17099	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17100	break;
17101	}
17102	#endif
17103
17104	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17105	hmR0VmxReadExitQualVmcs(pVmxTransient);
17106	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17107
17108	VMXVEXITINFO ExitInfo;
17109	RT_ZERO(ExitInfo);
17110	ExitInfo.uReason = pVmxTransient->uExitReason;
17111	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17112	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17113	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17114	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17115	}
17116
17117	/** @} */
17118	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17119

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 81150

Download in other formats: