HMVMXR0.cpp@ 81207

Last change on this file since 81207 was 81207, checked in by vboxsync, 5 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Build fix.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 708.9 KB

Line
1	/* $Id: HMVMXR0.cpp 81207 2019-10-10 11:12:13Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#include "HMInternal.h"
39	#include <VBox/vmm/vmcc.h>
40	#include <VBox/vmm/hmvmxinline.h>
41	#include "HMVMXR0.h"
42	#include "dtrace/VBoxVMM.h"
43
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
47	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/**
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
83	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
84
85	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
86	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
87	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
88	\| HMVMX_READ_EXIT_INSTR_LEN \
89	\| HMVMX_READ_IDT_VECTORING_INFO \
90	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
91
92	/** Assert that all the given fields have been read from the VMCS. */
93	#ifdef VBOX_STRICT
94	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
95	do { \
96	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
97	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); \
98	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
99	} while (0)
100	#else
101	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
102	#endif
103
104	/**
105	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
106	* guest using hardware-assisted VMX.
107	*
108	* This excludes state like GPRs (other than RSP) which are always are
109	* swapped and restored across the world-switch and also registers like EFER,
110	* MSR which cannot be modified by the guest without causing a VM-exit.
111	*/
112	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
113	\| CPUMCTX_EXTRN_RFLAGS \
114	\| CPUMCTX_EXTRN_RSP \
115	\| CPUMCTX_EXTRN_SREG_MASK \
116	\| CPUMCTX_EXTRN_TABLE_MASK \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_TSC_AUX \
121	\| CPUMCTX_EXTRN_OTHER_MSRS \
122	\| CPUMCTX_EXTRN_CR0 \
123	\| CPUMCTX_EXTRN_CR3 \
124	\| CPUMCTX_EXTRN_CR4 \
125	\| CPUMCTX_EXTRN_DR7 \
126	\| CPUMCTX_EXTRN_HWVIRT \
127	\| CPUMCTX_EXTRN_HM_VMX_MASK)
128
129	/**
130	* Exception bitmap mask for real-mode guests (real-on-v86).
131	*
132	* We need to intercept all exceptions manually except:
133	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
134	* due to bugs in Intel CPUs.
135	* - \#PF need not be intercepted even in real-mode if we have nested paging
136	* support.
137	*/
138	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
139	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
140	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
141	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
142	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
143	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
144	\| RT_BIT(X86_XCPT_XF))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158	/** Assert that preemption is disabled or covered by thread-context hooks. */
159	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
160	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
161
162	/** Assert that we haven't migrated CPUs when thread-context hooks are not
163	* used. */
164	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
165	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
166	("Illegal migration! Entered on CPU %u Current %u\n", \
167	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
168
169	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
170	* context. */
171	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
172	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
173	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
174
175	/** Log the VM-exit reason with an easily visible marker to identify it in a
176	* potential sea of logging data. */
177	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
178	do { \
179	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
180	HMGetVmxExitName(a_uExitReason))); \
181	} while (0) \
182
183
184	/*********************************************************************************************************************************
185	* Structures and Typedefs *
186	*********************************************************************************************************************************/
187	/**
188	* VMX per-VCPU transient state.
189	*
190	* A state structure for holding miscellaneous information across
191	* VMX non-root operation and restored after the transition.
192	*/
193	typedef struct VMXTRANSIENT
194	{
195	/** The host's rflags/eflags. */
196	RTCCUINTREG fEFlags;
197
198	/** The guest's TPR value used for TPR shadowing. */
199	uint8_t u8GuestTpr;
200	/** Alignment. */
201	uint8_t abAlignment0[7];
202
203	/** The basic VM-exit reason. */
204	uint16_t uExitReason;
205	/** Alignment. */
206	uint16_t u16Alignment0;
207	/** The VM-exit interruption error code. */
208	uint32_t uExitIntErrorCode;
209	/** The VM-exit exit code qualification. */
210	uint64_t uExitQual;
211	/** The Guest-linear address. */
212	uint64_t uGuestLinearAddr;
213	/** The Guest-physical address. */
214	uint64_t uGuestPhysicalAddr;
215	/** The Guest pending-debug exceptions. */
216	uint64_t uGuestPendingDbgXcpts;
217
218	/** The VM-exit interruption-information field. */
219	uint32_t uExitIntInfo;
220	/** The VM-exit instruction-length field. */
221	uint32_t cbExitInstr;
222	/** The VM-exit instruction-information field. */
223	VMXEXITINSTRINFO ExitInstrInfo;
224	/** Whether the VM-entry failed or not. */
225	bool fVMEntryFailed;
226	/** Whether we are currently executing a nested-guest. */
227	bool fIsNestedGuest;
228	/** Alignment. */
229	uint8_t abAlignment1[2];
230
231	/** The VM-entry interruption-information field. */
232	uint32_t uEntryIntInfo;
233	/** The VM-entry exception error code field. */
234	uint32_t uEntryXcptErrorCode;
235	/** The VM-entry instruction length field. */
236	uint32_t cbEntryInstr;
237
238	/** IDT-vectoring information field. */
239	uint32_t uIdtVectoringInfo;
240	/** IDT-vectoring error code. */
241	uint32_t uIdtVectoringErrorCode;
242
243	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
244	uint32_t fVmcsFieldsRead;
245
246	/** Whether the guest debug state was active at the time of VM-exit. */
247	bool fWasGuestDebugStateActive;
248	/** Whether the hyper debug state was active at the time of VM-exit. */
249	bool fWasHyperDebugStateActive;
250	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
251	bool fUpdatedTscOffsettingAndPreemptTimer;
252	/** Whether the VM-exit was caused by a page-fault during delivery of a
253	* contributory exception or a page-fault. */
254	bool fVectoringDoublePF;
255	/** Whether the VM-exit was caused by a page-fault during delivery of an
256	* external interrupt or NMI. */
257	bool fVectoringPF;
258	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
259	* area after VM-exit. */
260	bool fRemoveTscAuxMsr;
261	bool afAlignment0[2];
262
263	/** The VMCS info. object. */
264	PVMXVMCSINFO pVmcsInfo;
265	} VMXTRANSIENT;
266	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
267	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
268	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
269	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
271	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
272	/** Pointer to VMX transient state. */
273	typedef VMXTRANSIENT *PVMXTRANSIENT;
274	/** Pointer to a const VMX transient state. */
275	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
276
277	/**
278	* Memory operand read or write access.
279	*/
280	typedef enum VMXMEMACCESS
281	{
282	VMXMEMACCESS_READ = 0,
283	VMXMEMACCESS_WRITE = 1
284	} VMXMEMACCESS;
285
286	/**
287	* VMX VM-exit handler.
288	*
289	* @returns Strict VBox status code (i.e. informational status codes too).
290	* @param pVCpu The cross context virtual CPU structure.
291	* @param pVmxTransient The VMX-transient structure.
292	*/
293	#ifndef HMVMX_USE_FUNCTION_TABLE
294	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
295	#else
296	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
297	/** Pointer to VM-exit handler. */
298	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
299	#endif
300
301	/**
302	* VMX VM-exit handler, non-strict status code.
303	*
304	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
305	*
306	* @returns VBox status code, no informational status code returned.
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*
310	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
311	* use of that status code will be replaced with VINF_EM_SOMETHING
312	* later when switching over to IEM.
313	*/
314	#ifndef HMVMX_USE_FUNCTION_TABLE
315	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
316	#else
317	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
318	#endif
319
320
321	/*********************************************************************************************************************************
322	* Internal Functions *
323	*********************************************************************************************************************************/
324	#ifndef HMVMX_USE_FUNCTION_TABLE
325	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
326	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
327	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
328	#else
329	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
330	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
331	#endif
332	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
333	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
334	#endif
335
336	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
337
338	/** @name VM-exit handler prototypes.
339	* @{
340	*/
341	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
342	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
343	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
344	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
345	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
346	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
347	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
348	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
349	static FNVMXEXITHANDLER hmR0VmxExitHlt;
350	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
351	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
352	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
353	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
354	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
355	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
356	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
357	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
358	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
359	static FNVMXEXITHANDLER hmR0VmxExitVmread;
360	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
361	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
362	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
363	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
364	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
365	#endif
366	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
367	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
368	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
369	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
370	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
371	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
372	static FNVMXEXITHANDLER hmR0VmxExitMwait;
373	static FNVMXEXITHANDLER hmR0VmxExitMtf;
374	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
375	static FNVMXEXITHANDLER hmR0VmxExitPause;
376	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
377	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
378	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
379	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
380	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
381	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
382	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
383	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
384	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
385	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
386	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
388	/** @} */
389
390	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
391	/** @name Nested-guest VM-exit handler prototypes.
392	* @{
393	*/
394	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
395	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
398	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
399	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
400	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
401	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
402	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
403	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
404	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
405	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
406	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
407	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
408	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
409	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
410	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
411	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
412	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
414	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
415	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
416	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
417	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
418	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
419	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
421	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
422	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
423	/** @} */
424	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
425
426
427	/*********************************************************************************************************************************
428	* Global Variables *
429	*********************************************************************************************************************************/
430	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
431	/**
432	* Array of all VMCS fields.
433	* Any fields added to the VT-x spec. should be added here.
434	*
435	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
436	* of nested-guests.
437	*/
438	static const uint32_t g_aVmcsFields[] =
439	{
440	/* 16-bit control fields. */
441	VMX_VMCS16_VPID,
442	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
443	VMX_VMCS16_EPTP_INDEX,
444
445	/* 16-bit guest-state fields. */
446	VMX_VMCS16_GUEST_ES_SEL,
447	VMX_VMCS16_GUEST_CS_SEL,
448	VMX_VMCS16_GUEST_SS_SEL,
449	VMX_VMCS16_GUEST_DS_SEL,
450	VMX_VMCS16_GUEST_FS_SEL,
451	VMX_VMCS16_GUEST_GS_SEL,
452	VMX_VMCS16_GUEST_LDTR_SEL,
453	VMX_VMCS16_GUEST_TR_SEL,
454	VMX_VMCS16_GUEST_INTR_STATUS,
455	VMX_VMCS16_GUEST_PML_INDEX,
456
457	/* 16-bits host-state fields. */
458	VMX_VMCS16_HOST_ES_SEL,
459	VMX_VMCS16_HOST_CS_SEL,
460	VMX_VMCS16_HOST_SS_SEL,
461	VMX_VMCS16_HOST_DS_SEL,
462	VMX_VMCS16_HOST_FS_SEL,
463	VMX_VMCS16_HOST_GS_SEL,
464	VMX_VMCS16_HOST_TR_SEL,
465
466	/* 64-bit control fields. */
467	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
468	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
469	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
470	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
471	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
472	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
473	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
474	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
475	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
476	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
477	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
478	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
479	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
480	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
481	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
482	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
483	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
484	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
485	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
486	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
487	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
488	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
489	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
490	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
491	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
492	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
493	VMX_VMCS64_CTRL_EPTP_FULL,
494	VMX_VMCS64_CTRL_EPTP_HIGH,
495	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
496	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
497	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
498	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
499	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
500	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
501	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
502	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
503	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
504	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
505	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
506	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
507	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
508	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
509	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
510	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
511	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
512	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
513	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
514	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
515	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
516	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
517
518	/* 64-bit read-only data fields. */
519	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
520	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
521
522	/* 64-bit guest-state fields. */
523	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
524	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
525	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
526	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
527	VMX_VMCS64_GUEST_PAT_FULL,
528	VMX_VMCS64_GUEST_PAT_HIGH,
529	VMX_VMCS64_GUEST_EFER_FULL,
530	VMX_VMCS64_GUEST_EFER_HIGH,
531	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
532	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
533	VMX_VMCS64_GUEST_PDPTE0_FULL,
534	VMX_VMCS64_GUEST_PDPTE0_HIGH,
535	VMX_VMCS64_GUEST_PDPTE1_FULL,
536	VMX_VMCS64_GUEST_PDPTE1_HIGH,
537	VMX_VMCS64_GUEST_PDPTE2_FULL,
538	VMX_VMCS64_GUEST_PDPTE2_HIGH,
539	VMX_VMCS64_GUEST_PDPTE3_FULL,
540	VMX_VMCS64_GUEST_PDPTE3_HIGH,
541	VMX_VMCS64_GUEST_BNDCFGS_FULL,
542	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
543
544	/* 64-bit host-state fields. */
545	VMX_VMCS64_HOST_PAT_FULL,
546	VMX_VMCS64_HOST_PAT_HIGH,
547	VMX_VMCS64_HOST_EFER_FULL,
548	VMX_VMCS64_HOST_EFER_HIGH,
549	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
550	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
551
552	/* 32-bit control fields. */
553	VMX_VMCS32_CTRL_PIN_EXEC,
554	VMX_VMCS32_CTRL_PROC_EXEC,
555	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
556	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
557	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
558	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
559	VMX_VMCS32_CTRL_EXIT,
560	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
561	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
562	VMX_VMCS32_CTRL_ENTRY,
563	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
564	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
565	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
566	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
567	VMX_VMCS32_CTRL_TPR_THRESHOLD,
568	VMX_VMCS32_CTRL_PROC_EXEC2,
569	VMX_VMCS32_CTRL_PLE_GAP,
570	VMX_VMCS32_CTRL_PLE_WINDOW,
571
572	/* 32-bits read-only fields. */
573	VMX_VMCS32_RO_VM_INSTR_ERROR,
574	VMX_VMCS32_RO_EXIT_REASON,
575	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
576	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
577	VMX_VMCS32_RO_IDT_VECTORING_INFO,
578	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
579	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
580	VMX_VMCS32_RO_EXIT_INSTR_INFO,
581
582	/* 32-bit guest-state fields. */
583	VMX_VMCS32_GUEST_ES_LIMIT,
584	VMX_VMCS32_GUEST_CS_LIMIT,
585	VMX_VMCS32_GUEST_SS_LIMIT,
586	VMX_VMCS32_GUEST_DS_LIMIT,
587	VMX_VMCS32_GUEST_FS_LIMIT,
588	VMX_VMCS32_GUEST_GS_LIMIT,
589	VMX_VMCS32_GUEST_LDTR_LIMIT,
590	VMX_VMCS32_GUEST_TR_LIMIT,
591	VMX_VMCS32_GUEST_GDTR_LIMIT,
592	VMX_VMCS32_GUEST_IDTR_LIMIT,
593	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
594	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
595	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
596	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
597	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
598	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
599	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
600	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_INT_STATE,
602	VMX_VMCS32_GUEST_ACTIVITY_STATE,
603	VMX_VMCS32_GUEST_SMBASE,
604	VMX_VMCS32_GUEST_SYSENTER_CS,
605	VMX_VMCS32_PREEMPT_TIMER_VALUE,
606
607	/* 32-bit host-state fields. */
608	VMX_VMCS32_HOST_SYSENTER_CS,
609
610	/* Natural-width control fields. */
611	VMX_VMCS_CTRL_CR0_MASK,
612	VMX_VMCS_CTRL_CR4_MASK,
613	VMX_VMCS_CTRL_CR0_READ_SHADOW,
614	VMX_VMCS_CTRL_CR4_READ_SHADOW,
615	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
616	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
617	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
618	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
619
620	/* Natural-width read-only data fields. */
621	VMX_VMCS_RO_EXIT_QUALIFICATION,
622	VMX_VMCS_RO_IO_RCX,
623	VMX_VMCS_RO_IO_RSI,
624	VMX_VMCS_RO_IO_RDI,
625	VMX_VMCS_RO_IO_RIP,
626	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
627
628	/* Natural-width guest-state field */
629	VMX_VMCS_GUEST_CR0,
630	VMX_VMCS_GUEST_CR3,
631	VMX_VMCS_GUEST_CR4,
632	VMX_VMCS_GUEST_ES_BASE,
633	VMX_VMCS_GUEST_CS_BASE,
634	VMX_VMCS_GUEST_SS_BASE,
635	VMX_VMCS_GUEST_DS_BASE,
636	VMX_VMCS_GUEST_FS_BASE,
637	VMX_VMCS_GUEST_GS_BASE,
638	VMX_VMCS_GUEST_LDTR_BASE,
639	VMX_VMCS_GUEST_TR_BASE,
640	VMX_VMCS_GUEST_GDTR_BASE,
641	VMX_VMCS_GUEST_IDTR_BASE,
642	VMX_VMCS_GUEST_DR7,
643	VMX_VMCS_GUEST_RSP,
644	VMX_VMCS_GUEST_RIP,
645	VMX_VMCS_GUEST_RFLAGS,
646	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
647	VMX_VMCS_GUEST_SYSENTER_ESP,
648	VMX_VMCS_GUEST_SYSENTER_EIP,
649
650	/* Natural-width host-state fields */
651	VMX_VMCS_HOST_CR0,
652	VMX_VMCS_HOST_CR3,
653	VMX_VMCS_HOST_CR4,
654	VMX_VMCS_HOST_FS_BASE,
655	VMX_VMCS_HOST_GS_BASE,
656	VMX_VMCS_HOST_TR_BASE,
657	VMX_VMCS_HOST_GDTR_BASE,
658	VMX_VMCS_HOST_IDTR_BASE,
659	VMX_VMCS_HOST_SYSENTER_ESP,
660	VMX_VMCS_HOST_SYSENTER_EIP,
661	VMX_VMCS_HOST_RSP,
662	VMX_VMCS_HOST_RIP
663	};
664	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
665
666	static const uint32_t g_aVmcsSegBase[] =
667	{
668	VMX_VMCS_GUEST_ES_BASE,
669	VMX_VMCS_GUEST_CS_BASE,
670	VMX_VMCS_GUEST_SS_BASE,
671	VMX_VMCS_GUEST_DS_BASE,
672	VMX_VMCS_GUEST_FS_BASE,
673	VMX_VMCS_GUEST_GS_BASE
674	};
675	static const uint32_t g_aVmcsSegSel[] =
676	{
677	VMX_VMCS16_GUEST_ES_SEL,
678	VMX_VMCS16_GUEST_CS_SEL,
679	VMX_VMCS16_GUEST_SS_SEL,
680	VMX_VMCS16_GUEST_DS_SEL,
681	VMX_VMCS16_GUEST_FS_SEL,
682	VMX_VMCS16_GUEST_GS_SEL
683	};
684	static const uint32_t g_aVmcsSegLimit[] =
685	{
686	VMX_VMCS32_GUEST_ES_LIMIT,
687	VMX_VMCS32_GUEST_CS_LIMIT,
688	VMX_VMCS32_GUEST_SS_LIMIT,
689	VMX_VMCS32_GUEST_DS_LIMIT,
690	VMX_VMCS32_GUEST_FS_LIMIT,
691	VMX_VMCS32_GUEST_GS_LIMIT
692	};
693	static const uint32_t g_aVmcsSegAttr[] =
694	{
695	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
696	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
697	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
698	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
699	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
700	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
701	};
702	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
703	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
704	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
705	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
706
707	#ifdef HMVMX_USE_FUNCTION_TABLE
708	/**
709	* VMX_EXIT dispatch table.
710	*/
711	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
712	{
713	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
714	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
715	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
716	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
717	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
718	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
719	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
720	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
721	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
722	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
723	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
724	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
725	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
726	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
727	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
728	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
729	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
730	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
731	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
732	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
733	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
734	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
735	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
736	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
737	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
738	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
739	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
740	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
741	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
742	#else
743	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
744	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
745	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
746	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
747	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
748	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
749	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
750	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
751	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
752	#endif
753	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
754	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
755	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
756	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
757	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
758	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
759	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
760	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
761	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
762	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
763	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
764	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
765	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
766	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
767	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
768	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
769	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
770	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
771	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
772	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
773	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
774	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
775	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
776	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
777	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
778	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
779	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
780	#else
781	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
782	#endif
783	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
784	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
785	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
786	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
787	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
788	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
789	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
790	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
791	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
792	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
793	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
794	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
795	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
796	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
797	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
798	};
799	#endif /* HMVMX_USE_FUNCTION_TABLE */
800
801	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
802	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
803	{
804	/* 0 */ "(Not Used)",
805	/* 1 */ "VMCALL executed in VMX root operation.",
806	/* 2 */ "VMCLEAR with invalid physical address.",
807	/* 3 */ "VMCLEAR with VMXON pointer.",
808	/* 4 */ "VMLAUNCH with non-clear VMCS.",
809	/* 5 */ "VMRESUME with non-launched VMCS.",
810	/* 6 */ "VMRESUME after VMXOFF",
811	/* 7 */ "VM-entry with invalid control fields.",
812	/* 8 */ "VM-entry with invalid host state fields.",
813	/* 9 */ "VMPTRLD with invalid physical address.",
814	/* 10 */ "VMPTRLD with VMXON pointer.",
815	/* 11 */ "VMPTRLD with incorrect revision identifier.",
816	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
817	/* 13 */ "VMWRITE to read-only VMCS component.",
818	/* 14 */ "(Not Used)",
819	/* 15 */ "VMXON executed in VMX root operation.",
820	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
821	/* 17 */ "VM-entry with non-launched executing VMCS.",
822	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
823	/* 19 */ "VMCALL with non-clear VMCS.",
824	/* 20 */ "VMCALL with invalid VM-exit control fields.",
825	/* 21 */ "(Not Used)",
826	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
827	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
828	/* 24 */ "VMCALL with invalid SMM-monitor features.",
829	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
830	/* 26 */ "VM-entry with events blocked by MOV SS.",
831	/* 27 */ "(Not Used)",
832	/* 28 */ "Invalid operand to INVEPT/INVVPID."
833	};
834	#endif /* VBOX_STRICT && LOG_ENABLED */
835
836
837	/**
838	* Gets the CR0 guest/host mask.
839	*
840	* These bits typically does not change through the lifetime of a VM. Any bit set in
841	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
842	* by the guest.
843	*
844	* @returns The CR0 guest/host mask.
845	* @param pVCpu The cross context virtual CPU structure.
846	*/
847	static uint64_t hmR0VmxGetFixedCr0Mask(PCVMCPUCC pVCpu)
848	{
849	/*
850	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
851	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
852	*
853	* Furthermore, modifications to any bits that are reserved/unspecified currently
854	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
855	* when future CPUs specify and use currently reserved/unspecified bits.
856	*/
857	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
858	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
859	* and @bugref{6944}. */
860	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
861	return ( X86_CR0_PE
862	\| X86_CR0_NE
863	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
864	\| X86_CR0_PG
865	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
866	}
867
868
869	/**
870	* Gets the CR4 guest/host mask.
871	*
872	* These bits typically does not change through the lifetime of a VM. Any bit set in
873	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
874	* by the guest.
875	*
876	* @returns The CR4 guest/host mask.
877	* @param pVCpu The cross context virtual CPU structure.
878	*/
879	static uint64_t hmR0VmxGetFixedCr4Mask(PCVMCPUCC pVCpu)
880	{
881	/*
882	* We construct a mask of all CR4 bits that the guest can modify without causing
883	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
884	* a VM-exit when the guest attempts to modify them when executing using
885	* hardware-assisted VMX.
886	*
887	* When a feature is not exposed to the guest (and may be present on the host),
888	* we want to intercept guest modifications to the bit so we can emulate proper
889	* behavior (e.g., #GP).
890	*
891	* Furthermore, only modifications to those bits that don't require immediate
892	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
893	* depends on CR3 which might not always be the guest value while executing
894	* using hardware-assisted VMX.
895	*/
896	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
897	bool const fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
898	bool const fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
899	bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
900
901	/*
902	* Paranoia.
903	* Ensure features exposed to the guest are present on the host.
904	*/
905	Assert(!fFsGsBase \|\| pVM->cpum.ro.HostFeatures.fFsGsBase);
906	Assert(!fXSaveRstor \|\| pVM->cpum.ro.HostFeatures.fXSaveRstor);
907	Assert(!fFxSaveRstor \|\| pVM->cpum.ro.HostFeatures.fFxSaveRstor);
908
909	uint64_t const fGstMask = ( X86_CR4_PVI
910	\| X86_CR4_TSD
911	\| X86_CR4_DE
912	\| X86_CR4_MCE
913	\| X86_CR4_PCE
914	\| X86_CR4_OSXMMEEXCPT
915	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
916	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
917	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0));
918	return ~fGstMask;
919	}
920
921
922	/**
923	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
924	* area.
925	*
926	* @returns @c true if it's different, @c false otherwise.
927	* @param pVmcsInfo The VMCS info. object.
928	*/
929	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
930	{
931	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
932	&& pVmcsInfo->pvGuestMsrStore);
933	}
934
935
936	/**
937	* Sets the given Processor-based VM-execution controls.
938	*
939	* @param pVmxTransient The VMX-transient structure.
940	* @param uProcCtls The Processor-based VM-execution controls to set.
941	*/
942	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
943	{
944	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
945	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
946	{
947	pVmcsInfo->u32ProcCtls \|= uProcCtls;
948	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
949	AssertRC(rc);
950	}
951	}
952
953
954	/**
955	* Removes the given Processor-based VM-execution controls.
956	*
957	* @param pVCpu The cross context virtual CPU structure.
958	* @param pVmxTransient The VMX-transient structure.
959	* @param uProcCtls The Processor-based VM-execution controls to remove.
960	*
961	* @remarks When executing a nested-guest, this will not remove any of the specified
962	* controls if the nested hypervisor has set any one of them.
963	*/
964	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
965	{
966	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
967	if (pVmcsInfo->u32ProcCtls & uProcCtls)
968	{
969	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
970	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
971	? true
972	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
973	#else
974	NOREF(pVCpu);
975	bool const fRemoveCtls = true;
976	#endif
977	if (fRemoveCtls)
978	{
979	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
980	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
981	AssertRC(rc);
982	}
983	}
984	}
985
986
987	/**
988	* Sets the TSC offset for the current VMCS.
989	*
990	* @param uTscOffset The TSC offset to set.
991	* @param pVmcsInfo The VMCS info. object.
992	*/
993	static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
994	{
995	if (pVmcsInfo->u64TscOffset != uTscOffset)
996	{
997	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
998	AssertRC(rc);
999	pVmcsInfo->u64TscOffset = uTscOffset;
1000	}
1001	}
1002
1003
1004	/**
1005	* Adds one or more exceptions to the exception bitmap and commits it to the current
1006	* VMCS.
1007	*
1008	* @param pVmxTransient The VMX-transient structure.
1009	* @param uXcptMask The exception(s) to add.
1010	*/
1011	static void hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1012	{
1013	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1014	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1015	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1016	{
1017	uXcptBitmap \|= uXcptMask;
1018	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1019	AssertRC(rc);
1020	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1021	}
1022	}
1023
1024
1025	/**
1026	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1027	*
1028	* @param pVmxTransient The VMX-transient structure.
1029	* @param uXcpt The exception to add.
1030	*/
1031	static void hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1032	{
1033	Assert(uXcpt <= X86_XCPT_LAST);
1034	hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1035	}
1036
1037
1038	/**
1039	* Remove one or more exceptions from the exception bitmap and commits it to the
1040	* current VMCS.
1041	*
1042	* This takes care of not removing the exception intercept if a nested-guest
1043	* requires the exception to be intercepted.
1044	*
1045	* @returns VBox status code.
1046	* @param pVCpu The cross context virtual CPU structure.
1047	* @param pVmxTransient The VMX-transient structure.
1048	* @param uXcptMask The exception(s) to remove.
1049	*/
1050	static int hmR0VmxRemoveXcptInterceptMask(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1051	{
1052	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1053	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1054	if (u32XcptBitmap & uXcptMask)
1055	{
1056	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1057	if (!pVmxTransient->fIsNestedGuest)
1058	{ /* likely */ }
1059	else
1060	{
1061	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1062	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1063	}
1064	#endif
1065	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1066	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1067	\| RT_BIT(X86_XCPT_DE)
1068	\| RT_BIT(X86_XCPT_NM)
1069	\| RT_BIT(X86_XCPT_TS)
1070	\| RT_BIT(X86_XCPT_UD)
1071	\| RT_BIT(X86_XCPT_NP)
1072	\| RT_BIT(X86_XCPT_SS)
1073	\| RT_BIT(X86_XCPT_GP)
1074	\| RT_BIT(X86_XCPT_PF)
1075	\| RT_BIT(X86_XCPT_MF));
1076	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1077	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1078	#endif
1079	if (uXcptMask)
1080	{
1081	/* Validate we are not removing any essential exception intercepts. */
1082	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1083	NOREF(pVCpu);
1084	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1085	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1086
1087	/* Remove it from the exception bitmap. */
1088	u32XcptBitmap &= ~uXcptMask;
1089
1090	/* Commit and update the cache if necessary. */
1091	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1092	{
1093	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1094	AssertRC(rc);
1095	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1096	}
1097	}
1098	}
1099	return VINF_SUCCESS;
1100	}
1101
1102
1103	/**
1104	* Remove an exceptions from the exception bitmap and commits it to the current
1105	* VMCS.
1106	*
1107	* @returns VBox status code.
1108	* @param pVCpu The cross context virtual CPU structure.
1109	* @param pVmxTransient The VMX-transient structure.
1110	* @param uXcpt The exception to remove.
1111	*/
1112	static int hmR0VmxRemoveXcptIntercept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1113	{
1114	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1115	}
1116
1117
1118	/**
1119	* Loads the VMCS specified by the VMCS info. object.
1120	*
1121	* @returns VBox status code.
1122	* @param pVmcsInfo The VMCS info. object.
1123	*
1124	* @remarks Can be called with interrupts disabled.
1125	*/
1126	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1127	{
1128	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1129	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1130
1131	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1132	if (RT_SUCCESS(rc))
1133	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1134	return rc;
1135	}
1136
1137
1138	/**
1139	* Clears the VMCS specified by the VMCS info. object.
1140	*
1141	* @returns VBox status code.
1142	* @param pVmcsInfo The VMCS info. object.
1143	*
1144	* @remarks Can be called with interrupts disabled.
1145	*/
1146	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1147	{
1148	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1150
1151	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1152	if (RT_SUCCESS(rc))
1153	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1154	return rc;
1155	}
1156
1157
1158	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1159	/**
1160	* Loads the shadow VMCS specified by the VMCS info. object.
1161	*
1162	* @returns VBox status code.
1163	* @param pVmcsInfo The VMCS info. object.
1164	*
1165	* @remarks Can be called with interrupts disabled.
1166	*/
1167	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1168	{
1169	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1170	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1171
1172	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1173	if (RT_SUCCESS(rc))
1174	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1175	return rc;
1176	}
1177
1178
1179	/**
1180	* Clears the shadow VMCS specified by the VMCS info. object.
1181	*
1182	* @returns VBox status code.
1183	* @param pVmcsInfo The VMCS info. object.
1184	*
1185	* @remarks Can be called with interrupts disabled.
1186	*/
1187	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1188	{
1189	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1190	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1191
1192	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1193	if (RT_SUCCESS(rc))
1194	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1195	return rc;
1196	}
1197
1198
1199	/**
1200	* Switches from and to the specified VMCSes.
1201	*
1202	* @returns VBox status code.
1203	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1204	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1205	*
1206	* @remarks Called with interrupts disabled.
1207	*/
1208	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1209	{
1210	/*
1211	* Clear the VMCS we are switching out if it has not already been cleared.
1212	* This will sync any CPU internal data back to the VMCS.
1213	*/
1214	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1215	{
1216	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1217	if (RT_SUCCESS(rc))
1218	{
1219	/*
1220	* The shadow VMCS, if any, would not be active at this point since we
1221	* would have cleared it while importing the virtual hardware-virtualization
1222	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1223	* clear the shadow VMCS here, just assert for safety.
1224	*/
1225	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1226	}
1227	else
1228	return rc;
1229	}
1230
1231	/*
1232	* Clear the VMCS we are switching to if it has not already been cleared.
1233	* This will initialize the VMCS launch state to "clear" required for loading it.
1234	*
1235	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1236	*/
1237	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1238	{
1239	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1240	if (RT_SUCCESS(rc))
1241	{ /* likely */ }
1242	else
1243	return rc;
1244	}
1245
1246	/*
1247	* Finally, load the VMCS we are switching to.
1248	*/
1249	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1250	}
1251
1252
1253	/**
1254	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1255	* caller.
1256	*
1257	* @returns VBox status code.
1258	* @param pVCpu The cross context virtual CPU structure.
1259	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1260	* true) or guest VMCS (pass false).
1261	*/
1262	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1263	{
1264	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1265	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1266
1267	PVMXVMCSINFO pVmcsInfoFrom;
1268	PVMXVMCSINFO pVmcsInfoTo;
1269	if (fSwitchToNstGstVmcs)
1270	{
1271	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1272	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1273	}
1274	else
1275	{
1276	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1277	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1278	}
1279
1280	/*
1281	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1282	* preemption hook code path acquires the current VMCS.
1283	*/
1284	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1285
1286	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1287	if (RT_SUCCESS(rc))
1288	{
1289	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1290
1291	/*
1292	* If we are switching to a VMCS that was executed on a different host CPU or was
1293	* never executed before, flag that we need to export the host state before executing
1294	* guest/nested-guest code using hardware-assisted VMX.
1295	*
1296	* This could probably be done in a preemptible context since the preemption hook
1297	* will flag the necessary change in host context. However, since preemption is
1298	* already disabled and to avoid making assumptions about host specific code in
1299	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1300	* disabled.
1301	*/
1302	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1303	{ /* likely */ }
1304	else
1305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1306
1307	ASMSetFlags(fEFlags);
1308
1309	/*
1310	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1311	* flag that we need to update the host MSR values there. Even if we decide in the
1312	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1313	* if its content differs, we would have to update the host MSRs anyway.
1314	*/
1315	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1316	}
1317	else
1318	ASMSetFlags(fEFlags);
1319	return rc;
1320	}
1321	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1322
1323
1324	/**
1325	* Updates the VM's last error record.
1326	*
1327	* If there was a VMX instruction error, reads the error data from the VMCS and
1328	* updates VCPU's last error record as well.
1329	*
1330	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1331	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1332	* VERR_VMX_INVALID_VMCS_FIELD.
1333	* @param rc The error code.
1334	*/
1335	static void hmR0VmxUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
1336	{
1337	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1338	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1339	{
1340	AssertPtrReturnVoid(pVCpu);
1341	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1342	}
1343	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1344	}
1345
1346
1347	#ifdef VBOX_STRICT
1348	/**
1349	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1350	* transient structure.
1351	*
1352	* @param pVmxTransient The VMX-transient structure.
1353	*/
1354	DECLINLINE(void) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1355	{
1356	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1357	AssertRC(rc);
1358	}
1359
1360
1361	/**
1362	* Reads the VM-entry exception error code field from the VMCS into
1363	* the VMX transient structure.
1364	*
1365	* @param pVmxTransient The VMX-transient structure.
1366	*/
1367	DECLINLINE(void) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1368	{
1369	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1370	AssertRC(rc);
1371	}
1372
1373
1374	/**
1375	* Reads the VM-entry exception error code field from the VMCS into
1376	* the VMX transient structure.
1377	*
1378	* @param pVmxTransient The VMX-transient structure.
1379	*/
1380	DECLINLINE(void) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1381	{
1382	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1383	AssertRC(rc);
1384	}
1385	#endif /* VBOX_STRICT */
1386
1387
1388	/**
1389	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1390	* transient structure.
1391	*
1392	* @param pVmxTransient The VMX-transient structure.
1393	*/
1394	DECLINLINE(void) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1395	{
1396	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1397	{
1398	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1399	AssertRC(rc);
1400	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1401	}
1402	}
1403
1404
1405	/**
1406	* Reads the VM-exit interruption error code from the VMCS into the VMX
1407	* transient structure.
1408	*
1409	* @param pVmxTransient The VMX-transient structure.
1410	*/
1411	DECLINLINE(void) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1412	{
1413	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1414	{
1415	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1416	AssertRC(rc);
1417	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1418	}
1419	}
1420
1421
1422	/**
1423	* Reads the VM-exit instruction length field from the VMCS into the VMX
1424	* transient structure.
1425	*
1426	* @param pVmxTransient The VMX-transient structure.
1427	*/
1428	DECLINLINE(void) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1429	{
1430	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1431	{
1432	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1433	AssertRC(rc);
1434	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1435	}
1436	}
1437
1438
1439	/**
1440	* Reads the VM-exit instruction-information field from the VMCS into
1441	* the VMX transient structure.
1442	*
1443	* @param pVmxTransient The VMX-transient structure.
1444	*/
1445	DECLINLINE(void) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1446	{
1447	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1448	{
1449	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1450	AssertRC(rc);
1451	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1452	}
1453	}
1454
1455
1456	/**
1457	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1458	*
1459	* @param pVmxTransient The VMX-transient structure.
1460	*/
1461	DECLINLINE(void) hmR0VmxReadExitQualVmcs(PVMXTRANSIENT pVmxTransient)
1462	{
1463	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1464	{
1465	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1466	AssertRC(rc);
1467	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1468	}
1469	}
1470
1471
1472	/**
1473	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1474	*
1475	* @param pVmxTransient The VMX-transient structure.
1476	*/
1477	DECLINLINE(void) hmR0VmxReadGuestLinearAddrVmcs(PVMXTRANSIENT pVmxTransient)
1478	{
1479	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1480	{
1481	int rc = VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1482	AssertRC(rc);
1483	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1484	}
1485	}
1486
1487
1488	/**
1489	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1490	*
1491	* @param pVmxTransient The VMX-transient structure.
1492	*/
1493	DECLINLINE(void) hmR0VmxReadGuestPhysicalAddrVmcs(PVMXTRANSIENT pVmxTransient)
1494	{
1495	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1496	{
1497	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1498	AssertRC(rc);
1499	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1500	}
1501	}
1502
1503	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1504	/**
1505	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1506	* structure.
1507	*
1508	* @param pVmxTransient The VMX-transient structure.
1509	*/
1510	DECLINLINE(void) hmR0VmxReadGuestPendingDbgXctps(PVMXTRANSIENT pVmxTransient)
1511	{
1512	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1513	{
1514	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1515	AssertRC(rc);
1516	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1517	}
1518	}
1519	#endif
1520
1521	/**
1522	* Reads the IDT-vectoring information field from the VMCS into the VMX
1523	* transient structure.
1524	*
1525	* @param pVmxTransient The VMX-transient structure.
1526	*
1527	* @remarks No-long-jump zone!!!
1528	*/
1529	DECLINLINE(void) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1530	{
1531	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1532	{
1533	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1534	AssertRC(rc);
1535	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1536	}
1537	}
1538
1539
1540	/**
1541	* Reads the IDT-vectoring error code from the VMCS into the VMX
1542	* transient structure.
1543	*
1544	* @param pVmxTransient The VMX-transient structure.
1545	*/
1546	DECLINLINE(void) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1547	{
1548	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1549	{
1550	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1551	AssertRC(rc);
1552	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1553	}
1554	}
1555
1556	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1557	/**
1558	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1559	*
1560	* @param pVmxTransient The VMX-transient structure.
1561	*/
1562	static void hmR0VmxReadAllRoFieldsVmcs(PVMXTRANSIENT pVmxTransient)
1563	{
1564	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1565	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1566	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1567	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1568	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1569	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1570	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1571	rc \|= VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1572	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1573	AssertRC(rc);
1574	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1575	\| HMVMX_READ_EXIT_INSTR_LEN
1576	\| HMVMX_READ_EXIT_INSTR_INFO
1577	\| HMVMX_READ_IDT_VECTORING_INFO
1578	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1579	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1580	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1581	\| HMVMX_READ_GUEST_LINEAR_ADDR
1582	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1583	}
1584	#endif
1585
1586	/**
1587	* Enters VMX root mode operation on the current CPU.
1588	*
1589	* @returns VBox status code.
1590	* @param pHostCpu The HM physical-CPU structure.
1591	* @param pVM The cross context VM structure. Can be
1592	* NULL, after a resume.
1593	* @param HCPhysCpuPage Physical address of the VMXON region.
1594	* @param pvCpuPage Pointer to the VMXON region.
1595	*/
1596	static int hmR0VmxEnterRootMode(PHMPHYSCPU pHostCpu, PVMCC pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1597	{
1598	Assert(pHostCpu);
1599	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1600	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1601	Assert(pvCpuPage);
1602	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1603
1604	if (pVM)
1605	{
1606	/* Write the VMCS revision identifier to the VMXON region. */
1607	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1608	}
1609
1610	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1611	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1612
1613	/* Enable the VMX bit in CR4 if necessary. */
1614	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1615
1616	/* Record whether VMXE was already prior to us enabling it above. */
1617	pHostCpu->fVmxeAlreadyEnabled = RT_BOOL(uOldCr4 & X86_CR4_VMXE);
1618
1619	/* Enter VMX root mode. */
1620	int rc = VMXEnable(HCPhysCpuPage);
1621	if (RT_FAILURE(rc))
1622	{
1623	/* Restore CR4.VMXE if it was not set prior to our attempt to set it above. */
1624	if (!pHostCpu->fVmxeAlreadyEnabled)
1625	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1626
1627	if (pVM)
1628	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1629	}
1630
1631	/* Restore interrupts. */
1632	ASMSetFlags(fEFlags);
1633	return rc;
1634	}
1635
1636
1637	/**
1638	* Exits VMX root mode operation on the current CPU.
1639	*
1640	* @returns VBox status code.
1641	* @param pHostCpu The HM physical-CPU structure.
1642	*/
1643	static int hmR0VmxLeaveRootMode(PHMPHYSCPU pHostCpu)
1644	{
1645	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1646
1647	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1648	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1649
1650	/* If we're for some reason not in VMX root mode, then don't leave it. */
1651	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1652
1653	int rc;
1654	if (uHostCr4 & X86_CR4_VMXE)
1655	{
1656	/* Exit VMX root mode and clear the VMX bit in CR4. */
1657	VMXDisable();
1658
1659	/* Clear CR4.VMXE only if it was clear prior to use setting it. */
1660	if (!pHostCpu->fVmxeAlreadyEnabled)
1661	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1662
1663	rc = VINF_SUCCESS;
1664	}
1665	else
1666	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1667
1668	/* Restore interrupts. */
1669	ASMSetFlags(fEFlags);
1670	return rc;
1671	}
1672
1673
1674	/**
1675	* Allocates and maps a physically contiguous page. The allocated page is
1676	* zero'd out (used by various VT-x structures).
1677	*
1678	* @returns IPRT status code.
1679	* @param pMemObj Pointer to the ring-0 memory object.
1680	* @param ppVirt Where to store the virtual address of the allocation.
1681	* @param pHCPhys Where to store the physical address of the allocation.
1682	*/
1683	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1684	{
1685	AssertPtr(pMemObj);
1686	AssertPtr(ppVirt);
1687	AssertPtr(pHCPhys);
1688	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1689	if (RT_FAILURE(rc))
1690	return rc;
1691	ppVirt = RTR0MemObjAddress(pMemObj);
1692	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1693	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1694	return VINF_SUCCESS;
1695	}
1696
1697
1698	/**
1699	* Frees and unmaps an allocated, physical page.
1700	*
1701	* @param pMemObj Pointer to the ring-0 memory object.
1702	* @param ppVirt Where to re-initialize the virtual address of allocation as
1703	* 0.
1704	* @param pHCPhys Where to re-initialize the physical address of the
1705	* allocation as 0.
1706	*/
1707	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1708	{
1709	AssertPtr(pMemObj);
1710	AssertPtr(ppVirt);
1711	AssertPtr(pHCPhys);
1712	/* NULL is valid, accepted and ignored by the free function below. */
1713	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1714	*pMemObj = NIL_RTR0MEMOBJ;
1715	*ppVirt = NULL;
1716	*pHCPhys = NIL_RTHCPHYS;
1717	}
1718
1719
1720	/**
1721	* Initializes a VMCS info. object.
1722	*
1723	* @param pVmcsInfo The VMCS info. object.
1724	*/
1725	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1726	{
1727	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1728
1729	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1730	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1731	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1732	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1733	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1734	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1735	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1736	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1737	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1738	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1739	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1740	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1741	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1742	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1743	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1744	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
1745	pVmcsInfo->idHostCpuExec = NIL_RTCPUID;
1746	}
1747
1748
1749	/**
1750	* Frees the VT-x structures for a VMCS info. object.
1751	*
1752	* @param pVM The cross context VM structure.
1753	* @param pVmcsInfo The VMCS info. object.
1754	*/
1755	static void hmR0VmxFreeVmcsInfo(PVMCC pVM, PVMXVMCSINFO pVmcsInfo)
1756	{
1757	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1758
1759	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1760	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1761	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1762	#endif
1763
1764	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1765	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1766
1767	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1768	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1769	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1770
1771	hmR0VmxInitVmcsInfo(pVmcsInfo);
1772	}
1773
1774
1775	/**
1776	* Allocates the VT-x structures for a VMCS info. object.
1777	*
1778	* @returns VBox status code.
1779	* @param pVCpu The cross context virtual CPU structure.
1780	* @param pVmcsInfo The VMCS info. object.
1781	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1782	*/
1783	static int hmR0VmxAllocVmcsInfo(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1784	{
1785	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1786
1787	/* Allocate the guest VM control structure (VMCS). */
1788	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1789	if (RT_SUCCESS(rc))
1790	{
1791	if (!fIsNstGstVmcs)
1792	{
1793	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1794	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1795	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1796	#endif
1797	if (RT_SUCCESS(rc))
1798	{
1799	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1800	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1801	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1802	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1803	}
1804	}
1805	else
1806	{
1807	/* We don't yet support exposing VMCS shadowing to the guest. */
1808	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1809	Assert(!pVmcsInfo->pvShadowVmcs);
1810
1811	/* Get the allocated virtual-APIC page from CPUM. */
1812	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1813	{
1814	/** @todo NSTVMX: Get rid of this. There is no need to allocate a separate HC
1815	* page for this. Use the one provided by the nested-guest directly. */
1816	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1817	&pVmcsInfo->HCPhysVirtApic);
1818	Assert(pVmcsInfo->pbVirtApic);
1819	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1820	}
1821	}
1822
1823	if (RT_SUCCESS(rc))
1824	{
1825	/*
1826	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1827	* transparent accesses of specific MSRs.
1828	*
1829	* If the condition for enabling MSR bitmaps changes here, don't forget to
1830	* update HMIsMsrBitmapActive().
1831	*
1832	* We don't share MSR bitmaps between the guest and nested-guest as we then
1833	* don't need to care about carefully restoring the guest MSR bitmap.
1834	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1835	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1836	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1837	* we do that later while merging VMCS.
1838	*/
1839	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1840	{
1841	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1842	if ( RT_SUCCESS(rc)
1843	&& !fIsNstGstVmcs)
1844	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1845	}
1846
1847	if (RT_SUCCESS(rc))
1848	{
1849	/*
1850	* Allocate the VM-entry MSR-load area for the guest MSRs.
1851	*
1852	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1853	* the guest and nested-guest.
1854	*/
1855	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1856	&pVmcsInfo->HCPhysGuestMsrLoad);
1857	if (RT_SUCCESS(rc))
1858	{
1859	/*
1860	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1861	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1862	*/
1863	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1864	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1865	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1866
1867	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1868	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1869	&pVmcsInfo->HCPhysHostMsrLoad);
1870	}
1871	}
1872	}
1873	}
1874
1875	return rc;
1876	}
1877
1878
1879	/**
1880	* Free all VT-x structures for the VM.
1881	*
1882	* @returns IPRT status code.
1883	* @param pVM The cross context VM structure.
1884	*/
1885	static void hmR0VmxStructsFree(PVMCC pVM)
1886	{
1887	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1888	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1889	#endif
1890	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1891
1892	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1893	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1894	{
1895	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1896	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1897	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1898	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1899	}
1900	#endif
1901
1902	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1903	{
1904	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1905	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1906	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1907	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1908	if (pVM->cpum.ro.GuestFeatures.fVmx)
1909	{
1910	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1911	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1912	}
1913	#endif
1914	}
1915	}
1916
1917
1918	/**
1919	* Allocate all VT-x structures for the VM.
1920	*
1921	* @returns IPRT status code.
1922	* @param pVM The cross context VM structure.
1923	*/
1924	static int hmR0VmxStructsAlloc(PVMCC pVM)
1925	{
1926	/*
1927	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1928	* The VMCS size cannot be more than 4096 bytes.
1929	*
1930	* See Intel spec. Appendix A.1 "Basic VMX Information".
1931	*/
1932	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1933	if (cbVmcs <= X86_PAGE_4K_SIZE)
1934	{ /* likely */ }
1935	else
1936	{
1937	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1938	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1939	}
1940
1941	/*
1942	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1943	*/
1944	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1945	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1946	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1947	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1948	#endif
1949
1950	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1951	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1952	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1953
1954	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1955	{
1956	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1957	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1958	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1959	}
1960
1961	/*
1962	* Allocate per-VM VT-x structures.
1963	*/
1964	int rc = VINF_SUCCESS;
1965	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1966	/* Allocate crash-dump magic scratch page. */
1967	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1968	if (RT_FAILURE(rc))
1969	{
1970	hmR0VmxStructsFree(pVM);
1971	return rc;
1972	}
1973	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1974	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1975	#endif
1976
1977	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1978	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1979	{
1980	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1981	&pVM->hm.s.vmx.HCPhysApicAccess);
1982	if (RT_FAILURE(rc))
1983	{
1984	hmR0VmxStructsFree(pVM);
1985	return rc;
1986	}
1987	}
1988
1989	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1990	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1991	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1992	{
1993	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1994	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1995	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1996	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1997	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1998	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1999	{
2000	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
2001	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
2002	if (RT_SUCCESS(rc))
2003	{
2004	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
2005	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
2006	}
2007	}
2008	else
2009	rc = VERR_NO_MEMORY;
2010
2011	if (RT_FAILURE(rc))
2012	{
2013	hmR0VmxStructsFree(pVM);
2014	return rc;
2015	}
2016	}
2017	#endif
2018
2019	/*
2020	* Initialize per-VCPU VT-x structures.
2021	*/
2022	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2023	{
2024	/* Allocate the guest VMCS structures. */
2025	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
2026	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2027	if (RT_SUCCESS(rc))
2028	{
2029	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2030	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2031	if (pVM->cpum.ro.GuestFeatures.fVmx)
2032	{
2033	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2034	if (RT_SUCCESS(rc))
2035	{ /* likely */ }
2036	else
2037	break;
2038	}
2039	#endif
2040	}
2041	else
2042	break;
2043	}
2044
2045	if (RT_FAILURE(rc))
2046	{
2047	hmR0VmxStructsFree(pVM);
2048	return rc;
2049	}
2050
2051	return VINF_SUCCESS;
2052	}
2053
2054	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2055	/**
2056	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2057	*
2058	* @returns @c true if the MSR is intercepted, @c false otherwise.
2059	* @param pvMsrBitmap The MSR bitmap.
2060	* @param offMsr The MSR byte offset.
2061	* @param iBit The bit offset from the byte offset.
2062	*/
2063	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2064	{
2065	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2066	Assert(pbMsrBitmap);
2067	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2068	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2069	}
2070	#endif
2071
2072	/**
2073	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2074	*
2075	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2076	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2077	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2078	* the read/write access of this MSR.
2079	*
2080	* @param pVCpu The cross context virtual CPU structure.
2081	* @param pVmcsInfo The VMCS info. object.
2082	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2083	* @param idMsr The MSR value.
2084	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2085	* include both a read -and- a write permission!
2086	*
2087	* @sa CPUMGetVmxMsrPermission.
2088	* @remarks Can be called with interrupts disabled.
2089	*/
2090	static void hmR0VmxSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2091	{
2092	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2093	Assert(pbMsrBitmap);
2094	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2095
2096	/*
2097	* MSR-bitmap Layout:
2098	* Byte index MSR range Interpreted as
2099	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2100	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2101	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2102	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2103	*
2104	* A bit corresponding to an MSR within the above range causes a VM-exit
2105	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2106	* the MSR range, it always cause a VM-exit.
2107	*
2108	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2109	*/
2110	uint16_t const offBitmapRead = 0;
2111	uint16_t const offBitmapWrite = 0x800;
2112	uint16_t offMsr;
2113	int32_t iBit;
2114	if (idMsr <= UINT32_C(0x00001fff))
2115	{
2116	offMsr = 0;
2117	iBit = idMsr;
2118	}
2119	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2120	{
2121	offMsr = 0x400;
2122	iBit = idMsr - UINT32_C(0xc0000000);
2123	}
2124	else
2125	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2126
2127	/*
2128	* Set the MSR read permission.
2129	*/
2130	uint16_t const offMsrRead = offBitmapRead + offMsr;
2131	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2132	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2133	{
2134	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2135	bool const fClear = !fIsNstGstVmcs ? true
2136	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2137	#else
2138	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2139	bool const fClear = true;
2140	#endif
2141	if (fClear)
2142	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2143	}
2144	else
2145	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2146
2147	/*
2148	* Set the MSR write permission.
2149	*/
2150	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2151	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2152	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2153	{
2154	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2155	bool const fClear = !fIsNstGstVmcs ? true
2156	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2157	#else
2158	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2159	bool const fClear = true;
2160	#endif
2161	if (fClear)
2162	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2163	}
2164	else
2165	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2166	}
2167
2168
2169	/**
2170	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2171	* area.
2172	*
2173	* @returns VBox status code.
2174	* @param pVCpu The cross context virtual CPU structure.
2175	* @param pVmcsInfo The VMCS info. object.
2176	* @param cMsrs The number of MSRs.
2177	*/
2178	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2179	{
2180	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2181	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2182	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2183	{
2184	/* Commit the MSR counts to the VMCS and update the cache. */
2185	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2186	{
2187	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2188	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
2189	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2190	pVmcsInfo->cEntryMsrLoad = cMsrs;
2191	pVmcsInfo->cExitMsrStore = cMsrs;
2192	pVmcsInfo->cExitMsrLoad = cMsrs;
2193	}
2194	return VINF_SUCCESS;
2195	}
2196
2197	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2198	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2199	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2200	}
2201
2202
2203	/**
2204	* Adds a new (or updates the value of an existing) guest/host MSR
2205	* pair to be swapped during the world-switch as part of the
2206	* auto-load/store MSR area in the VMCS.
2207	*
2208	* @returns VBox status code.
2209	* @param pVCpu The cross context virtual CPU structure.
2210	* @param pVmxTransient The VMX-transient structure.
2211	* @param idMsr The MSR.
2212	* @param uGuestMsrValue Value of the guest MSR.
2213	* @param fSetReadWrite Whether to set the guest read/write access of this
2214	* MSR (thus not causing a VM-exit).
2215	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2216	* necessary.
2217	*/
2218	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2219	bool fSetReadWrite, bool fUpdateHostMsr)
2220	{
2221	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2222	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2223	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2224	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2225	uint32_t i;
2226
2227	/* Paranoia. */
2228	Assert(pGuestMsrLoad);
2229
2230	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2231
2232	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2233	for (i = 0; i < cMsrs; i++)
2234	{
2235	if (pGuestMsrLoad[i].u32Msr == idMsr)
2236	break;
2237	}
2238
2239	bool fAdded = false;
2240	if (i == cMsrs)
2241	{
2242	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2243	++cMsrs;
2244	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2245	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2246
2247	/* Set the guest to read/write this MSR without causing VM-exits. */
2248	if ( fSetReadWrite
2249	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2250	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2251
2252	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2253	fAdded = true;
2254	}
2255
2256	/* Update the MSR value for the newly added or already existing MSR. */
2257	pGuestMsrLoad[i].u32Msr = idMsr;
2258	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2259
2260	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2261	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2262	{
2263	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2264	pGuestMsrStore[i].u32Msr = idMsr;
2265	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2266	}
2267
2268	/* Update the corresponding slot in the host MSR area. */
2269	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2270	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2271	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2272	pHostMsr[i].u32Msr = idMsr;
2273
2274	/*
2275	* Only if the caller requests to update the host MSR value AND we've newly added the
2276	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2277	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2278	*
2279	* We do this for performance reasons since reading MSRs may be quite expensive.
2280	*/
2281	if (fAdded)
2282	{
2283	if (fUpdateHostMsr)
2284	{
2285	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2286	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2287	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2288	}
2289	else
2290	{
2291	/* Someone else can do the work. */
2292	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2293	}
2294	}
2295	return VINF_SUCCESS;
2296	}
2297
2298
2299	/**
2300	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2301	* auto-load/store MSR area in the VMCS.
2302	*
2303	* @returns VBox status code.
2304	* @param pVCpu The cross context virtual CPU structure.
2305	* @param pVmxTransient The VMX-transient structure.
2306	* @param idMsr The MSR.
2307	*/
2308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2309	{
2310	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2311	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2312	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2313	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2314
2315	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2316
2317	for (uint32_t i = 0; i < cMsrs; i++)
2318	{
2319	/* Find the MSR. */
2320	if (pGuestMsrLoad[i].u32Msr == idMsr)
2321	{
2322	/*
2323	* If it's the last MSR, we only need to reduce the MSR count.
2324	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2325	*/
2326	if (i < cMsrs - 1)
2327	{
2328	/* Remove it from the VM-entry MSR-load area. */
2329	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2330	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2331
2332	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2333	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2334	{
2335	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2336	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2337	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2338	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2339	}
2340
2341	/* Remove it from the VM-exit MSR-load area. */
2342	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2343	Assert(pHostMsr[i].u32Msr == idMsr);
2344	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2345	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2346	}
2347
2348	/* Reduce the count to reflect the removed MSR and bail. */
2349	--cMsrs;
2350	break;
2351	}
2352	}
2353
2354	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2355	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2356	{
2357	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2358	AssertRCReturn(rc, rc);
2359
2360	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2361	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2362	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2363
2364	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2365	return VINF_SUCCESS;
2366	}
2367
2368	return VERR_NOT_FOUND;
2369	}
2370
2371
2372	/**
2373	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2374	*
2375	* @returns @c true if found, @c false otherwise.
2376	* @param pVmcsInfo The VMCS info. object.
2377	* @param idMsr The MSR to find.
2378	*/
2379	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2380	{
2381	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2382	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2383	Assert(pMsrs);
2384	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2385	for (uint32_t i = 0; i < cMsrs; i++)
2386	{
2387	if (pMsrs[i].u32Msr == idMsr)
2388	return true;
2389	}
2390	return false;
2391	}
2392
2393
2394	/**
2395	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2396	*
2397	* @param pVCpu The cross context virtual CPU structure.
2398	* @param pVmcsInfo The VMCS info. object.
2399	*
2400	* @remarks No-long-jump zone!!!
2401	*/
2402	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2403	{
2404	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2405
2406	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2407	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2408	Assert(pHostMsrLoad);
2409	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2410	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2411	for (uint32_t i = 0; i < cMsrs; i++)
2412	{
2413	/*
2414	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2415	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2416	*/
2417	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2418	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2419	else
2420	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2421	}
2422	}
2423
2424
2425	/**
2426	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2427	* perform lazy restoration of the host MSRs while leaving VT-x.
2428	*
2429	* @param pVCpu The cross context virtual CPU structure.
2430	*
2431	* @remarks No-long-jump zone!!!
2432	*/
2433	static void hmR0VmxLazySaveHostMsrs(PVMCPUCC pVCpu)
2434	{
2435	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2436
2437	/*
2438	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2439	*/
2440	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2441	{
2442	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2443	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2444	{
2445	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2446	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2447	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2448	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2449	}
2450	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2451	}
2452	}
2453
2454
2455	/**
2456	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2457	* lazily while leaving VT-x.
2458	*
2459	* @returns true if it does, false otherwise.
2460	* @param pVCpu The cross context virtual CPU structure.
2461	* @param idMsr The MSR to check.
2462	*/
2463	static bool hmR0VmxIsLazyGuestMsr(PCVMCPUCC pVCpu, uint32_t idMsr)
2464	{
2465	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2466	{
2467	switch (idMsr)
2468	{
2469	case MSR_K8_LSTAR:
2470	case MSR_K6_STAR:
2471	case MSR_K8_SF_MASK:
2472	case MSR_K8_KERNEL_GS_BASE:
2473	return true;
2474	}
2475	}
2476	return false;
2477	}
2478
2479
2480	/**
2481	* Loads a set of guests MSRs to allow read/passthru to the guest.
2482	*
2483	* The name of this function is slightly confusing. This function does NOT
2484	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2485	* common prefix for functions dealing with "lazy restoration" of the shared
2486	* MSRs.
2487	*
2488	* @param pVCpu The cross context virtual CPU structure.
2489	*
2490	* @remarks No-long-jump zone!!!
2491	*/
2492	static void hmR0VmxLazyLoadGuestMsrs(PVMCPUCC pVCpu)
2493	{
2494	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2495	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2496
2497	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2498	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2499	{
2500	/*
2501	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2502	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2503	* we can skip a few MSR writes.
2504	*
2505	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2506	* guest MSR values in the guest-CPU context might be different to what's currently
2507	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2508	* CPU, see @bugref{8728}.
2509	*/
2510	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2511	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2512	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2513	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2514	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2515	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2516	{
2517	#ifdef VBOX_STRICT
2518	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2519	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2520	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2521	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2522	#endif
2523	}
2524	else
2525	{
2526	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2527	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2528	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2529	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2530	}
2531	}
2532	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2533	}
2534
2535
2536	/**
2537	* Performs lazy restoration of the set of host MSRs if they were previously
2538	* loaded with guest MSR values.
2539	*
2540	* @param pVCpu The cross context virtual CPU structure.
2541	*
2542	* @remarks No-long-jump zone!!!
2543	* @remarks The guest MSRs should have been saved back into the guest-CPU
2544	* context by hmR0VmxImportGuestState()!!!
2545	*/
2546	static void hmR0VmxLazyRestoreHostMsrs(PVMCPUCC pVCpu)
2547	{
2548	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2549	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2550
2551	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2552	{
2553	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2554	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2555	{
2556	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2557	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2558	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2559	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2560	}
2561	}
2562	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2563	}
2564
2565
2566	/**
2567	* Verifies that our cached values of the VMCS fields are all consistent with
2568	* what's actually present in the VMCS.
2569	*
2570	* @returns VBox status code.
2571	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2572	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2573	* VMCS content. HMCPU error-field is
2574	* updated, see VMX_VCI_XXX.
2575	* @param pVCpu The cross context virtual CPU structure.
2576	* @param pVmcsInfo The VMCS info. object.
2577	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2578	*/
2579	static int hmR0VmxCheckVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2580	{
2581	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2582
2583	uint32_t u32Val;
2584	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2585	AssertRC(rc);
2586	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2587	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2588	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2589	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2590
2591	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2592	AssertRC(rc);
2593	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2594	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2595	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2596	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2597
2598	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2599	AssertRC(rc);
2600	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2601	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2602	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2603	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2604
2605	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2606	AssertRC(rc);
2607	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2608	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2609	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2610	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2611
2612	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2613	{
2614	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2615	AssertRC(rc);
2616	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2617	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2618	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2619	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2620	}
2621
2622	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2623	AssertRC(rc);
2624	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2625	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2626	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2627	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2628
2629	uint64_t u64Val;
2630	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2631	AssertRC(rc);
2632	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2633	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2634	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2635	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2636
2637	NOREF(pcszVmcs);
2638	return VINF_SUCCESS;
2639	}
2640
2641
2642	#ifdef VBOX_STRICT
2643	/**
2644	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2645	*
2646	* @param pVCpu The cross context virtual CPU structure.
2647	* @param pVmcsInfo The VMCS info. object.
2648	*/
2649	static void hmR0VmxCheckHostEferMsr(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2650	{
2651	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2652
2653	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2654	{
2655	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2656	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2657	uint64_t uVmcsEferMsrVmcs;
2658	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2659	AssertRC(rc);
2660
2661	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2662	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2663	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2664	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2665	}
2666	}
2667
2668
2669	/**
2670	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2671	* VMCS are correct.
2672	*
2673	* @param pVCpu The cross context virtual CPU structure.
2674	* @param pVmcsInfo The VMCS info. object.
2675	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2676	*/
2677	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2678	{
2679	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2680
2681	/* Read the various MSR-area counts from the VMCS. */
2682	uint32_t cEntryLoadMsrs;
2683	uint32_t cExitStoreMsrs;
2684	uint32_t cExitLoadMsrs;
2685	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2686	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2687	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2688
2689	/* Verify all the MSR counts are the same. */
2690	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2691	Assert(cExitStoreMsrs == cExitLoadMsrs);
2692	uint32_t const cMsrs = cExitLoadMsrs;
2693
2694	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2695	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2696
2697	/* Verify the MSR counts are within the allocated page size. */
2698	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2699
2700	/* Verify the relevant contents of the MSR areas match. */
2701	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2702	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2703	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2704	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2705	for (uint32_t i = 0; i < cMsrs; i++)
2706	{
2707	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2708	if (fSeparateExitMsrStorePage)
2709	{
2710	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2711	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2712	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2713	}
2714
2715	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2716	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2717	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2718
2719	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2720	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2721	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2722	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2723
2724	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2725	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2726	if (fIsEferMsr)
2727	{
2728	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2729	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2730	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2731	}
2732
2733	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2734	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2735	{
2736	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2737	if (fIsEferMsr)
2738	{
2739	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2740	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2741	}
2742	else
2743	{
2744	if (!fIsNstGstVmcs)
2745	{
2746	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2747	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2748	}
2749	else
2750	{
2751	/*
2752	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2753	* execute a nested-guest with MSR passthrough.
2754	*
2755	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2756	* allow passthrough too.
2757	*/
2758	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2759	Assert(pvMsrBitmapNstGst);
2760	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2761	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2762	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2763	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2764	}
2765	}
2766	}
2767
2768	/* Move to the next MSR. */
2769	pHostMsrLoad++;
2770	pGuestMsrLoad++;
2771	pGuestMsrStore++;
2772	}
2773	}
2774	#endif /* VBOX_STRICT */
2775
2776
2777	/**
2778	* Flushes the TLB using EPT.
2779	*
2780	* @returns VBox status code.
2781	* @param pVCpu The cross context virtual CPU structure of the calling
2782	* EMT. Can be NULL depending on @a enmTlbFlush.
2783	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2784	* enmTlbFlush.
2785	* @param enmTlbFlush Type of flush.
2786	*
2787	* @remarks Caller is responsible for making sure this function is called only
2788	* when NestedPaging is supported and providing @a enmTlbFlush that is
2789	* supported by the CPU.
2790	* @remarks Can be called with interrupts disabled.
2791	*/
2792	static void hmR0VmxFlushEpt(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2793	{
2794	uint64_t au64Descriptor[2];
2795	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2796	au64Descriptor[0] = 0;
2797	else
2798	{
2799	Assert(pVCpu);
2800	Assert(pVmcsInfo);
2801	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2802	}
2803	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2804
2805	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2806	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2807
2808	if ( RT_SUCCESS(rc)
2809	&& pVCpu)
2810	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2811	}
2812
2813
2814	/**
2815	* Flushes the TLB using VPID.
2816	*
2817	* @returns VBox status code.
2818	* @param pVCpu The cross context virtual CPU structure of the calling
2819	* EMT. Can be NULL depending on @a enmTlbFlush.
2820	* @param enmTlbFlush Type of flush.
2821	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2822	* on @a enmTlbFlush).
2823	*
2824	* @remarks Can be called with interrupts disabled.
2825	*/
2826	static void hmR0VmxFlushVpid(PVMCPUCC pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2827	{
2828	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2829
2830	uint64_t au64Descriptor[2];
2831	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2832	{
2833	au64Descriptor[0] = 0;
2834	au64Descriptor[1] = 0;
2835	}
2836	else
2837	{
2838	AssertPtr(pVCpu);
2839	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2840	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2841	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2842	au64Descriptor[1] = GCPtr;
2843	}
2844
2845	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2846	AssertMsg(rc == VINF_SUCCESS,
2847	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2848
2849	if ( RT_SUCCESS(rc)
2850	&& pVCpu)
2851	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2852	NOREF(rc);
2853	}
2854
2855
2856	/**
2857	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2858	* otherwise there is nothing really to invalidate.
2859	*
2860	* @returns VBox status code.
2861	* @param pVCpu The cross context virtual CPU structure.
2862	* @param GCVirt Guest virtual address of the page to invalidate.
2863	*/
2864	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
2865	{
2866	AssertPtr(pVCpu);
2867	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2868
2869	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2870	{
2871	/*
2872	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2873	* the EPT case. See @bugref{6043} and @bugref{6177}.
2874	*
2875	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2876	* as this function maybe called in a loop with individual addresses.
2877	*/
2878	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2879	if (pVM->hm.s.vmx.fVpid)
2880	{
2881	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2882	if (fVpidFlush)
2883	{
2884	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2885	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2886	}
2887	else
2888	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2889	}
2890	else if (pVM->hm.s.fNestedPaging)
2891	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2892	}
2893
2894	return VINF_SUCCESS;
2895	}
2896
2897
2898	/**
2899	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2900	* case where neither EPT nor VPID is supported by the CPU.
2901	*
2902	* @param pHostCpu The HM physical-CPU structure.
2903	* @param pVCpu The cross context virtual CPU structure.
2904	*
2905	* @remarks Called with interrupts disabled.
2906	*/
2907	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2908	{
2909	AssertPtr(pVCpu);
2910	AssertPtr(pHostCpu);
2911
2912	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2913
2914	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2915	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2916	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2917	pVCpu->hm.s.fForceTLBFlush = false;
2918	return;
2919	}
2920
2921
2922	/**
2923	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2924	*
2925	* @param pHostCpu The HM physical-CPU structure.
2926	* @param pVCpu The cross context virtual CPU structure.
2927	* @param pVmcsInfo The VMCS info. object.
2928	*
2929	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2930	* nomenclature. The reason is, to avoid confusion in compare statements
2931	* since the host-CPU copies are named "ASID".
2932	*
2933	* @remarks Called with interrupts disabled.
2934	*/
2935	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2936	{
2937	#ifdef VBOX_WITH_STATISTICS
2938	bool fTlbFlushed = false;
2939	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2940	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2941	if (!fTlbFlushed) \
2942	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2943	} while (0)
2944	#else
2945	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2946	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2947	#endif
2948
2949	AssertPtr(pVCpu);
2950	AssertPtr(pHostCpu);
2951	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2952
2953	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2954	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2955	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2956	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2957
2958	/*
2959	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2960	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2961	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2962	* cannot reuse the current ASID anymore.
2963	*/
2964	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2965	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2966	{
2967	++pHostCpu->uCurrentAsid;
2968	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2969	{
2970	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2971	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2972	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2973	}
2974
2975	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2976	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2977	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2978
2979	/*
2980	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2981	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2982	*/
2983	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2984	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2985	HMVMX_SET_TAGGED_TLB_FLUSHED();
2986	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2987	}
2988	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2989	{
2990	/*
2991	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2992	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2993	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2994	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2995	* mappings, see @bugref{6568}.
2996	*
2997	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2998	*/
2999	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3000	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3001	HMVMX_SET_TAGGED_TLB_FLUSHED();
3002	}
3003	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3004	{
3005	/*
3006	* The nested-guest specifies its own guest-physical address to use as the APIC-access
3007	* address which requires flushing the TLB of EPT cached structures.
3008	*
3009	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
3010	*/
3011	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3012	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3013	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3014	HMVMX_SET_TAGGED_TLB_FLUSHED();
3015	}
3016
3017
3018	pVCpu->hm.s.fForceTLBFlush = false;
3019	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3020
3021	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3022	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3023	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3024	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3025	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3026	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3027	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3028	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3029	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3030
3031	/* Update VMCS with the VPID. */
3032	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3033	AssertRC(rc);
3034
3035	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3036	}
3037
3038
3039	/**
3040	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3041	*
3042	* @param pHostCpu The HM physical-CPU structure.
3043	* @param pVCpu The cross context virtual CPU structure.
3044	* @param pVmcsInfo The VMCS info. object.
3045	*
3046	* @remarks Called with interrupts disabled.
3047	*/
3048	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3049	{
3050	AssertPtr(pVCpu);
3051	AssertPtr(pHostCpu);
3052	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3053	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3054	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3055
3056	/*
3057	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3058	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3059	*/
3060	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3061	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3062	{
3063	pVCpu->hm.s.fForceTLBFlush = true;
3064	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3065	}
3066
3067	/* Check for explicit TLB flushes. */
3068	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3069	{
3070	pVCpu->hm.s.fForceTLBFlush = true;
3071	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3072	}
3073
3074	/* Check for TLB flushes while switching to/from a nested-guest. */
3075	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3076	{
3077	pVCpu->hm.s.fForceTLBFlush = true;
3078	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3079	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3080	}
3081
3082	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3083	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3084
3085	if (pVCpu->hm.s.fForceTLBFlush)
3086	{
3087	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3088	pVCpu->hm.s.fForceTLBFlush = false;
3089	}
3090	}
3091
3092
3093	/**
3094	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3095	*
3096	* @param pHostCpu The HM physical-CPU structure.
3097	* @param pVCpu The cross context virtual CPU structure.
3098	*
3099	* @remarks Called with interrupts disabled.
3100	*/
3101	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
3102	{
3103	AssertPtr(pVCpu);
3104	AssertPtr(pHostCpu);
3105	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3106	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3107	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3108
3109	/*
3110	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3111	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3112	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3113	* cannot reuse the current ASID anymore.
3114	*/
3115	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3116	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3117	{
3118	pVCpu->hm.s.fForceTLBFlush = true;
3119	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3120	}
3121
3122	/* Check for explicit TLB flushes. */
3123	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3124	{
3125	/*
3126	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3127	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3128	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3129	* include fExplicitFlush's too) - an obscure corner case.
3130	*/
3131	pVCpu->hm.s.fForceTLBFlush = true;
3132	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3133	}
3134
3135	/* Check for TLB flushes while switching to/from a nested-guest. */
3136	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3137	{
3138	pVCpu->hm.s.fForceTLBFlush = true;
3139	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3140	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3141	}
3142
3143	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3144	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3145	if (pVCpu->hm.s.fForceTLBFlush)
3146	{
3147	++pHostCpu->uCurrentAsid;
3148	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3149	{
3150	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3151	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3152	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3153	}
3154
3155	pVCpu->hm.s.fForceTLBFlush = false;
3156	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3157	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3158	if (pHostCpu->fFlushAsidBeforeUse)
3159	{
3160	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3161	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3162	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3163	{
3164	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3165	pHostCpu->fFlushAsidBeforeUse = false;
3166	}
3167	else
3168	{
3169	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3170	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3171	}
3172	}
3173	}
3174
3175	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3176	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3177	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3178	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3179	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3180	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3181	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3182
3183	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3184	AssertRC(rc);
3185	}
3186
3187
3188	/**
3189	* Flushes the guest TLB entry based on CPU capabilities.
3190	*
3191	* @param pHostCpu The HM physical-CPU structure.
3192	* @param pVCpu The cross context virtual CPU structure.
3193	* @param pVmcsInfo The VMCS info. object.
3194	*
3195	* @remarks Called with interrupts disabled.
3196	*/
3197	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3198	{
3199	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3200	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3201	#endif
3202	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3203	switch (pVM->hm.s.vmx.enmTlbFlushType)
3204	{
3205	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3206	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3207	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3208	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3209	default:
3210	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3211	break;
3212	}
3213	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3214	}
3215
3216
3217	/**
3218	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3219	* TLB entries from the host TLB before VM-entry.
3220	*
3221	* @returns VBox status code.
3222	* @param pVM The cross context VM structure.
3223	*/
3224	static int hmR0VmxSetupTaggedTlb(PVMCC pVM)
3225	{
3226	/*
3227	* Determine optimal flush type for nested paging.
3228	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3229	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3230	*/
3231	if (pVM->hm.s.fNestedPaging)
3232	{
3233	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3234	{
3235	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3236	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3237	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3238	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3239	else
3240	{
3241	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3242	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3243	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3244	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3245	}
3246
3247	/* Make sure the write-back cacheable memory type for EPT is supported. */
3248	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3249	{
3250	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3251	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3252	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3253	}
3254
3255	/* EPT requires a page-walk length of 4. */
3256	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3257	{
3258	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3259	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3260	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3261	}
3262	}
3263	else
3264	{
3265	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3266	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3267	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3268	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3269	}
3270	}
3271
3272	/*
3273	* Determine optimal flush type for VPID.
3274	*/
3275	if (pVM->hm.s.vmx.fVpid)
3276	{
3277	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3278	{
3279	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3280	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3281	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3282	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3283	else
3284	{
3285	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3286	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3287	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3288	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3289	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3290	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3291	pVM->hm.s.vmx.fVpid = false;
3292	}
3293	}
3294	else
3295	{
3296	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3297	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3298	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3299	pVM->hm.s.vmx.fVpid = false;
3300	}
3301	}
3302
3303	/*
3304	* Setup the handler for flushing tagged-TLBs.
3305	*/
3306	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3307	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3308	else if (pVM->hm.s.fNestedPaging)
3309	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3310	else if (pVM->hm.s.vmx.fVpid)
3311	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3312	else
3313	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3314	return VINF_SUCCESS;
3315	}
3316
3317
3318	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3319	/**
3320	* Sets up the shadow VMCS fields arrays.
3321	*
3322	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3323	* executing the guest.
3324	*
3325	* @returns VBox status code.
3326	* @param pVM The cross context VM structure.
3327	*/
3328	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVMCC pVM)
3329	{
3330	/*
3331	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3332	* when the host does not support it.
3333	*/
3334	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3335	if ( !fGstVmwriteAll
3336	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3337	{ /* likely. */ }
3338	else
3339	{
3340	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3341	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3342	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3343	}
3344
3345	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3346	uint32_t cRwFields = 0;
3347	uint32_t cRoFields = 0;
3348	for (uint32_t i = 0; i < cVmcsFields; i++)
3349	{
3350	VMXVMCSFIELD VmcsField;
3351	VmcsField.u = g_aVmcsFields[i];
3352
3353	/*
3354	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3355	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3356	* in the shadow VMCS fields array as they would be redundant.
3357	*
3358	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3359	* we must not include it in the shadow VMCS fields array. Guests attempting to
3360	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3361	* the required behavior.
3362	*/
3363	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3364	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3365	{
3366	/*
3367	* Read-only fields are placed in a separate array so that while syncing shadow
3368	* VMCS fields later (which is more performance critical) we can avoid branches.
3369	*
3370	* However, if the guest can write to all fields (including read-only fields),
3371	* we treat it a as read/write field. Otherwise, writing to these fields would
3372	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3373	*/
3374	if ( fGstVmwriteAll
3375	\|\| !VMXIsVmcsFieldReadOnly(VmcsField.u))
3376	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3377	else
3378	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3379	}
3380	}
3381
3382	/* Update the counts. */
3383	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3384	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3385	return VINF_SUCCESS;
3386	}
3387
3388
3389	/**
3390	* Sets up the VMREAD and VMWRITE bitmaps.
3391	*
3392	* @param pVM The cross context VM structure.
3393	*/
3394	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVMCC pVM)
3395	{
3396	/*
3397	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3398	*/
3399	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3400	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3401	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3402	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3403	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3404
3405	/*
3406	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3407	* VMREAD and VMWRITE bitmaps.
3408	*/
3409	{
3410	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3411	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3412	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3413	{
3414	uint32_t const uVmcsField = paShadowVmcsFields[i];
3415	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3416	Assert(uVmcsField >> 3 < cbBitmap);
3417	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3418	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3419	}
3420	}
3421
3422	/*
3423	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3424	* if the host supports VMWRITE to all supported VMCS fields.
3425	*/
3426	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3427	{
3428	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3429	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3430	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3431	{
3432	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3433	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3434	Assert(uVmcsField >> 3 < cbBitmap);
3435	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3436	}
3437	}
3438	}
3439	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3440
3441
3442	/**
3443	* Sets up the virtual-APIC page address for the VMCS.
3444	*
3445	* @param pVmcsInfo The VMCS info. object.
3446	*/
3447	DECLINLINE(void) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
3448	{
3449	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3450	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3451	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3452	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3453	AssertRC(rc);
3454	}
3455
3456
3457	/**
3458	* Sets up the MSR-bitmap address for the VMCS.
3459	*
3460	* @param pVmcsInfo The VMCS info. object.
3461	*/
3462	DECLINLINE(void) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
3463	{
3464	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3465	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3466	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3467	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3468	AssertRC(rc);
3469	}
3470
3471
3472	/**
3473	* Sets up the APIC-access page address for the VMCS.
3474	*
3475	* @param pVCpu The cross context virtual CPU structure.
3476	*/
3477	DECLINLINE(void) hmR0VmxSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
3478	{
3479	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3480	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3481	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3482	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3483	AssertRC(rc);
3484	}
3485
3486
3487	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3488	/**
3489	* Sets up the VMREAD bitmap address for the VMCS.
3490	*
3491	* @param pVCpu The cross context virtual CPU structure.
3492	*/
3493	DECLINLINE(void) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
3494	{
3495	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3496	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3497	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3498	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3499	AssertRC(rc);
3500	}
3501
3502
3503	/**
3504	* Sets up the VMWRITE bitmap address for the VMCS.
3505	*
3506	* @param pVCpu The cross context virtual CPU structure.
3507	*/
3508	DECLINLINE(void) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
3509	{
3510	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3511	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3512	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3513	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3514	AssertRC(rc);
3515	}
3516	#endif
3517
3518
3519	/**
3520	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3521	* in the VMCS.
3522	*
3523	* @returns VBox status code.
3524	* @param pVmcsInfo The VMCS info. object.
3525	*/
3526	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
3527	{
3528	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3529	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3530	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3531
3532	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3533	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3534	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3535
3536	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3537	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3538	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3539
3540	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad); AssertRC(rc);
3541	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore); AssertRC(rc);
3542	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad); AssertRC(rc);
3543	return VINF_SUCCESS;
3544	}
3545
3546
3547	/**
3548	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3549	*
3550	* @param pVCpu The cross context virtual CPU structure.
3551	* @param pVmcsInfo The VMCS info. object.
3552	*/
3553	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3554	{
3555	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3556
3557	/*
3558	* The guest can access the following MSRs (read, write) without causing
3559	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3560	*/
3561	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3562	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3563	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3564	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3565	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3566	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3567
3568	/*
3569	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3570	* associated with then. We never need to intercept access (writes need to be
3571	* executed without causing a VM-exit, reads will #GP fault anyway).
3572	*
3573	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3574	* read/write them. We swap the the guest/host MSR value using the
3575	* auto-load/store MSR area.
3576	*/
3577	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3578	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3579	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3580	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3581	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3582	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3583
3584	/*
3585	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3586	* required for 64-bit guests.
3587	*/
3588	if (pVM->hm.s.fAllow64BitGuests)
3589	{
3590	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3591	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3592	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3593	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3594	}
3595
3596	/*
3597	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3598	*/
3599	#ifdef VBOX_STRICT
3600	Assert(pVmcsInfo->pvMsrBitmap);
3601	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3602	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3603	#endif
3604	}
3605
3606
3607	/**
3608	* Sets up pin-based VM-execution controls in the VMCS.
3609	*
3610	* @returns VBox status code.
3611	* @param pVCpu The cross context virtual CPU structure.
3612	* @param pVmcsInfo The VMCS info. object.
3613	*/
3614	static int hmR0VmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3615	{
3616	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3617	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3618	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3619
3620	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3621	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3622
3623	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3624	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3625
3626	/* Enable the VMX-preemption timer. */
3627	if (pVM->hm.s.vmx.fUsePreemptTimer)
3628	{
3629	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3630	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3631	}
3632
3633	#if 0
3634	/* Enable posted-interrupt processing. */
3635	if (pVM->hm.s.fPostedIntrs)
3636	{
3637	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3638	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3639	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3640	}
3641	#endif
3642
3643	if ((fVal & fZap) != fVal)
3644	{
3645	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3646	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3647	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3648	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3649	}
3650
3651	/* Commit it to the VMCS and update our cache. */
3652	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3653	AssertRC(rc);
3654	pVmcsInfo->u32PinCtls = fVal;
3655
3656	return VINF_SUCCESS;
3657	}
3658
3659
3660	/**
3661	* Sets up secondary processor-based VM-execution controls in the VMCS.
3662	*
3663	* @returns VBox status code.
3664	* @param pVCpu The cross context virtual CPU structure.
3665	* @param pVmcsInfo The VMCS info. object.
3666	*/
3667	static int hmR0VmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3668	{
3669	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3670	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3671	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3672
3673	/* WBINVD causes a VM-exit. */
3674	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3675	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3676
3677	/* Enable EPT (aka nested-paging). */
3678	if (pVM->hm.s.fNestedPaging)
3679	fVal \|= VMX_PROC_CTLS2_EPT;
3680
3681	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3682	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3683	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3684	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3685	fVal \|= VMX_PROC_CTLS2_INVPCID;
3686
3687	/* Enable VPID. */
3688	if (pVM->hm.s.vmx.fVpid)
3689	fVal \|= VMX_PROC_CTLS2_VPID;
3690
3691	/* Enable unrestricted guest execution. */
3692	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3693	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3694
3695	#if 0
3696	if (pVM->hm.s.fVirtApicRegs)
3697	{
3698	/* Enable APIC-register virtualization. */
3699	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3700	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3701
3702	/* Enable virtual-interrupt delivery. */
3703	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3704	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3705	}
3706	#endif
3707
3708	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3709	where the TPR shadow resides. */
3710	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3711	* done dynamically. */
3712	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3713	{
3714	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3715	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3716	}
3717
3718	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3719	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3720	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3721	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3722	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3723
3724	/* Enable Pause-Loop exiting. */
3725	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3726	&& pVM->hm.s.vmx.cPleGapTicks
3727	&& pVM->hm.s.vmx.cPleWindowTicks)
3728	{
3729	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3730
3731	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
3732	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
3733	}
3734
3735	if ((fVal & fZap) != fVal)
3736	{
3737	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3738	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3739	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3740	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3741	}
3742
3743	/* Commit it to the VMCS and update our cache. */
3744	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3745	AssertRC(rc);
3746	pVmcsInfo->u32ProcCtls2 = fVal;
3747
3748	return VINF_SUCCESS;
3749	}
3750
3751
3752	/**
3753	* Sets up processor-based VM-execution controls in the VMCS.
3754	*
3755	* @returns VBox status code.
3756	* @param pVCpu The cross context virtual CPU structure.
3757	* @param pVmcsInfo The VMCS info. object.
3758	*/
3759	static int hmR0VmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3760	{
3761	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3762
3763	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3764	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3765
3766	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3767	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3768	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3769	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3770	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3771	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3772	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3773
3774	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3775	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3776	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3777	{
3778	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3779	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3780	}
3781
3782	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3783	if (!pVM->hm.s.fNestedPaging)
3784	{
3785	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3786	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3787	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3788	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3789	}
3790
3791	/* Use TPR shadowing if supported by the CPU. */
3792	if ( PDMHasApic(pVM)
3793	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3794	{
3795	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3796	/* CR8 writes cause a VM-exit based on TPR threshold. */
3797	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3798	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3799	hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3800	}
3801	else
3802	{
3803	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3804	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3805	if (pVM->hm.s.fAllow64BitGuests)
3806	{
3807	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3808	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3809	}
3810	}
3811
3812	/* Use MSR-bitmaps if supported by the CPU. */
3813	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3814	{
3815	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3816	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3817	}
3818
3819	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3820	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3821	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3822
3823	if ((fVal & fZap) != fVal)
3824	{
3825	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3826	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3827	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3828	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3829	}
3830
3831	/* Commit it to the VMCS and update our cache. */
3832	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3833	AssertRC(rc);
3834	pVmcsInfo->u32ProcCtls = fVal;
3835
3836	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3837	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3838	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3839
3840	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3841	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3842	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3843
3844	/* Sanity check, should not really happen. */
3845	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3846	{ /* likely */ }
3847	else
3848	{
3849	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3850	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3851	}
3852
3853	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3854	return VINF_SUCCESS;
3855	}
3856
3857
3858	/**
3859	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3860	* Processor-based VM-execution) control fields in the VMCS.
3861	*
3862	* @returns VBox status code.
3863	* @param pVCpu The cross context virtual CPU structure.
3864	* @param pVmcsInfo The VMCS info. object.
3865	*/
3866	static int hmR0VmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3867	{
3868	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3869	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3870	{
3871	hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3872	hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3873	}
3874	#endif
3875
3876	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3877	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3878	AssertRC(rc);
3879
3880	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3881	if (RT_SUCCESS(rc))
3882	{
3883	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3884	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3885
3886	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
3887	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
3888
3889	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3890	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3891	return VINF_SUCCESS;
3892	}
3893	else
3894	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3895	return rc;
3896	}
3897
3898
3899	/**
3900	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3901	*
3902	* We shall setup those exception intercepts that don't change during the
3903	* lifetime of the VM here. The rest are done dynamically while loading the
3904	* guest state.
3905	*
3906	* @param pVCpu The cross context virtual CPU structure.
3907	* @param pVmcsInfo The VMCS info. object.
3908	*/
3909	static void hmR0VmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3910	{
3911	/*
3912	* The following exceptions are always intercepted:
3913	*
3914	* #AC - To prevent the guest from hanging the CPU.
3915	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3916	* recursive #DBs can cause a CPU hang.
3917	* #PF - To sync our shadow page tables when nested-paging is not used.
3918	*/
3919	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3920	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3921	\| RT_BIT(X86_XCPT_DB)
3922	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3923
3924	/* Commit it to the VMCS. */
3925	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3926	AssertRC(rc);
3927
3928	/* Update our cache of the exception bitmap. */
3929	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3930	}
3931
3932
3933	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3934	/**
3935	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3936	*
3937	* @returns VBox status code.
3938	* @param pVCpu The cross context virtual CPU structure.
3939	* @param pVmcsInfo The VMCS info. object.
3940	*/
3941	static int hmR0VmxSetupVmcsCtlsNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3942	{
3943	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3944	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3945	AssertRC(rc);
3946
3947	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3948	if (RT_SUCCESS(rc))
3949	{
3950	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3951	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3952
3953	/* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
3954	Assert(!pVmcsInfo->u64Cr0Mask);
3955	Assert(!pVmcsInfo->u64Cr4Mask);
3956	return VINF_SUCCESS;
3957	}
3958	else
3959	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3960	return rc;
3961	}
3962	#endif
3963
3964
3965	/**
3966	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3967	* VMX.
3968	*
3969	* @returns VBox status code.
3970	* @param pVCpu The cross context virtual CPU structure.
3971	* @param pVmcsInfo The VMCS info. object.
3972	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3973	*/
3974	static int hmR0VmxSetupVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3975	{
3976	Assert(pVmcsInfo->pvVmcs);
3977	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3978
3979	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3980	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3981	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3982	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3983
3984	LogFlowFunc(("\n"));
3985
3986	/*
3987	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3988	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3989	*/
3990	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3991	if (RT_SUCCESS(rc))
3992	{
3993	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3994	if (RT_SUCCESS(rc))
3995	{
3996	if (!fIsNstGstVmcs)
3997	{
3998	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3999	if (RT_SUCCESS(rc))
4000	{
4001	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4002	if (RT_SUCCESS(rc))
4003	{
4004	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4005	if (RT_SUCCESS(rc))
4006	{
4007	hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4008	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4009	/*
4010	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4011	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4012	* making it fit for use when VMCS shadowing is later enabled.
4013	*/
4014	if (pVmcsInfo->pvShadowVmcs)
4015	{
4016	VMXVMCSREVID VmcsRevId;
4017	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4018	VmcsRevId.n.fIsShadowVmcs = 1;
4019	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4020	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4021	if (RT_SUCCESS(rc))
4022	{ /* likely */ }
4023	else
4024	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4025	}
4026	#endif
4027	}
4028	else
4029	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4030	}
4031	else
4032	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4033	}
4034	else
4035	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4036	}
4037	else
4038	{
4039	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4040	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4041	if (RT_SUCCESS(rc))
4042	{ /* likely */ }
4043	else
4044	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4045	#else
4046	AssertFailed();
4047	#endif
4048	}
4049	}
4050	else
4051	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4052	}
4053	else
4054	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4055
4056	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4057	if (RT_SUCCESS(rc))
4058	{
4059	rc = hmR0VmxClearVmcs(pVmcsInfo);
4060	if (RT_SUCCESS(rc))
4061	{ /* likely */ }
4062	else
4063	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4064	}
4065
4066	/*
4067	* Update the last-error record both for failures and success, so we
4068	* can propagate the status code back to ring-3 for diagnostics.
4069	*/
4070	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4071	NOREF(pszVmcs);
4072	return rc;
4073	}
4074
4075
4076	/**
4077	* Does global VT-x initialization (called during module initialization).
4078	*
4079	* @returns VBox status code.
4080	*/
4081	VMMR0DECL(int) VMXR0GlobalInit(void)
4082	{
4083	#ifdef HMVMX_USE_FUNCTION_TABLE
4084	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4085	# ifdef VBOX_STRICT
4086	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4087	Assert(g_apfnVMExitHandlers[i]);
4088	# endif
4089	#endif
4090	return VINF_SUCCESS;
4091	}
4092
4093
4094	/**
4095	* Does global VT-x termination (called during module termination).
4096	*/
4097	VMMR0DECL(void) VMXR0GlobalTerm()
4098	{
4099	/* Nothing to do currently. */
4100	}
4101
4102
4103	/**
4104	* Sets up and activates VT-x on the current CPU.
4105	*
4106	* @returns VBox status code.
4107	* @param pHostCpu The HM physical-CPU structure.
4108	* @param pVM The cross context VM structure. Can be
4109	* NULL after a host resume operation.
4110	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4111	* fEnabledByHost is @c true).
4112	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4113	* @a fEnabledByHost is @c true).
4114	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4115	* enable VT-x on the host.
4116	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4117	*/
4118	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4119	PCSUPHWVIRTMSRS pHwvirtMsrs)
4120	{
4121	AssertPtr(pHostCpu);
4122	AssertPtr(pHwvirtMsrs);
4123	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4124
4125	/* Enable VT-x if it's not already enabled by the host. */
4126	if (!fEnabledByHost)
4127	{
4128	int rc = hmR0VmxEnterRootMode(pHostCpu, pVM, HCPhysCpuPage, pvCpuPage);
4129	if (RT_FAILURE(rc))
4130	return rc;
4131	}
4132
4133	/*
4134	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4135	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4136	* invalidated when flushing by VPID.
4137	*/
4138	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4139	{
4140	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4141	pHostCpu->fFlushAsidBeforeUse = false;
4142	}
4143	else
4144	pHostCpu->fFlushAsidBeforeUse = true;
4145
4146	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4147	++pHostCpu->cTlbFlushes;
4148
4149	return VINF_SUCCESS;
4150	}
4151
4152
4153	/**
4154	* Deactivates VT-x on the current CPU.
4155	*
4156	* @returns VBox status code.
4157	* @param pHostCpu The HM physical-CPU structure.
4158	* @param pvCpuPage Pointer to the VMXON region.
4159	* @param HCPhysCpuPage Physical address of the VMXON region.
4160	*
4161	* @remarks This function should never be called when SUPR0EnableVTx() or
4162	* similar was used to enable VT-x on the host.
4163	*/
4164	VMMR0DECL(int) VMXR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4165	{
4166	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4167
4168	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4169	return hmR0VmxLeaveRootMode(pHostCpu);
4170	}
4171
4172
4173	/**
4174	* Does per-VM VT-x initialization.
4175	*
4176	* @returns VBox status code.
4177	* @param pVM The cross context VM structure.
4178	*/
4179	VMMR0DECL(int) VMXR0InitVM(PVMCC pVM)
4180	{
4181	AssertPtr(pVM);
4182	LogFlowFunc(("pVM=%p\n", pVM));
4183
4184	int rc = hmR0VmxStructsAlloc(pVM);
4185	if (RT_FAILURE(rc))
4186	{
4187	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4188	return rc;
4189	}
4190
4191	return VINF_SUCCESS;
4192	}
4193
4194
4195	/**
4196	* Does per-VM VT-x termination.
4197	*
4198	* @returns VBox status code.
4199	* @param pVM The cross context VM structure.
4200	*/
4201	VMMR0DECL(int) VMXR0TermVM(PVMCC pVM)
4202	{
4203	AssertPtr(pVM);
4204	LogFlowFunc(("pVM=%p\n", pVM));
4205
4206	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4207	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4208	{
4209	Assert(pVM->hm.s.vmx.pvScratch);
4210	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4211	}
4212	#endif
4213	hmR0VmxStructsFree(pVM);
4214	return VINF_SUCCESS;
4215	}
4216
4217
4218	/**
4219	* Sets up the VM for execution using hardware-assisted VMX.
4220	* This function is only called once per-VM during initialization.
4221	*
4222	* @returns VBox status code.
4223	* @param pVM The cross context VM structure.
4224	*/
4225	VMMR0DECL(int) VMXR0SetupVM(PVMCC pVM)
4226	{
4227	AssertPtr(pVM);
4228	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4229
4230	LogFlowFunc(("pVM=%p\n", pVM));
4231
4232	/*
4233	* At least verify if VMX is enabled, since we can't check if we're in VMX root mode or not
4234	* without causing a #GP.
4235	*/
4236	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4237	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4238	{ /* likely */ }
4239	else
4240	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4241
4242	/*
4243	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4244	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4245	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4246	*/
4247	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4248	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4249	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4250	{
4251	LogRelFunc(("Invalid real-on-v86 state.\n"));
4252	return VERR_INTERNAL_ERROR;
4253	}
4254
4255	/* Initialize these always, see hmR3InitFinalizeR0().*/
4256	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4257	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4258
4259	/* Setup the tagged-TLB flush handlers. */
4260	int rc = hmR0VmxSetupTaggedTlb(pVM);
4261	if (RT_FAILURE(rc))
4262	{
4263	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4264	return rc;
4265	}
4266
4267	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4268	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4269	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4270	{
4271	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4272	if (RT_SUCCESS(rc))
4273	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4274	else
4275	{
4276	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4277	return rc;
4278	}
4279	}
4280	#endif
4281
4282	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4283	{
4284	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
4285	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4286
4287	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4288	if (RT_SUCCESS(rc))
4289	{
4290	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4291	if (pVM->cpum.ro.GuestFeatures.fVmx)
4292	{
4293	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4294	if (RT_SUCCESS(rc))
4295	{ /* likely */ }
4296	else
4297	{
4298	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4299	return rc;
4300	}
4301	}
4302	#endif
4303	}
4304	else
4305	{
4306	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4307	return rc;
4308	}
4309	}
4310
4311	return VINF_SUCCESS;
4312	}
4313
4314
4315	/**
4316	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4317	* the VMCS.
4318	*/
4319	static void hmR0VmxExportHostControlRegs(void)
4320	{
4321	int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, ASMGetCR0()); AssertRC(rc);
4322	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, ASMGetCR3()); AssertRC(rc);
4323	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, ASMGetCR4()); AssertRC(rc);
4324	}
4325
4326
4327	/**
4328	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4329	* the host-state area in the VMCS.
4330	*
4331	* @returns VBox status code.
4332	* @param pVCpu The cross context virtual CPU structure.
4333	*/
4334	static int hmR0VmxExportHostSegmentRegs(PVMCPUCC pVCpu)
4335	{
4336	/**
4337	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4338	* requirements. See hmR0VmxExportHostSegmentRegs().
4339	*/
4340	#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_selValue) \
4341	if ((a_selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4342	{ \
4343	bool fValidSelector = true; \
4344	if ((a_selValue) & X86_SEL_LDT) \
4345	{ \
4346	uint32_t const uAttr = ASMGetSegAttr(a_selValue); \
4347	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4348	} \
4349	if (fValidSelector) \
4350	{ \
4351	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##a_Seg; \
4352	pVCpu->hm.s.vmx.RestoreHost.uHostSel##a_Seg = (a_selValue); \
4353	} \
4354	(a_selValue) = 0; \
4355	}
4356
4357	/*
4358	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4359	* will be messed up. We should -not- save the messed up state without restoring
4360	* the original host-state, see @bugref{7240}.
4361	*
4362	* This apparently can happen (most likely the FPU changes), deal with it rather than
4363	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4364	*/
4365	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4366	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4367	{
4368	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4369	pVCpu->idCpu));
4370	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4371	}
4372	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4373
4374	/*
4375	* Host segment registers.
4376	*/
4377	RTSEL uSelES = ASMGetES();
4378	RTSEL uSelCS = ASMGetCS();
4379	RTSEL uSelSS = ASMGetSS();
4380	RTSEL uSelDS = ASMGetDS();
4381	RTSEL uSelFS = ASMGetFS();
4382	RTSEL uSelGS = ASMGetGS();
4383	RTSEL uSelTR = ASMGetTR();
4384
4385	/*
4386	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4387	* gain VM-entry and restore them before we get preempted.
4388	*
4389	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4390	*/
4391	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4392	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4393	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4394	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4395
4396	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4397	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4398	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4399	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4400	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4401	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4402	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4403	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4404	Assert(uSelCS);
4405	Assert(uSelTR);
4406
4407	/* Write these host selector fields into the host-state area in the VMCS. */
4408	int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS); AssertRC(rc);
4409	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS); AssertRC(rc);
4410	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS); AssertRC(rc);
4411	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES); AssertRC(rc);
4412	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS); AssertRC(rc);
4413	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS); AssertRC(rc);
4414	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR); AssertRC(rc);
4415
4416	/*
4417	* Host GDTR and IDTR.
4418	*/
4419	RTGDTR Gdtr;
4420	RTIDTR Idtr;
4421	RT_ZERO(Gdtr);
4422	RT_ZERO(Idtr);
4423	ASMGetGDTR(&Gdtr);
4424	ASMGetIDTR(&Idtr);
4425	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRC(rc);
4426	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRC(rc);
4427
4428	/*
4429	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4430	* them to the maximum limit (0xffff) on every VM-exit.
4431	*/
4432	if (Gdtr.cbGdt != 0xffff)
4433	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4434
4435	/*
4436	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4437	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4438	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4439	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4440	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4441	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4442	* at 0xffff on hosts where we are sure it won't cause trouble.
4443	*/
4444	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4445	if (Idtr.cbIdt < 0x0fff)
4446	#else
4447	if (Idtr.cbIdt != 0xffff)
4448	#endif
4449	{
4450	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4451	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4452	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4453	}
4454
4455	/*
4456	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4457	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4458	* RPL should be too in most cases.
4459	*/
4460	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4461	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4462
4463	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4464	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4465
4466	/*
4467	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4468	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4469	* restoration if the host has something else. Task switching is not supported in 64-bit
4470	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4471	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4472	*
4473	* [1] See Intel spec. 3.5 "System Descriptor Types".
4474	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4475	*/
4476	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4477	Assert(pDesc->System.u4Type == 11);
4478	if ( pDesc->System.u16LimitLow != 0x67
4479	\|\| pDesc->System.u4LimitHigh)
4480	{
4481	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4482	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4483	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4484	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4485	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4486	}
4487
4488	/*
4489	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4490	*/
4491	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4492	{
4493	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4494	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4495	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4496	{
4497	/* The GDT is read-only but the writable GDT is available. */
4498	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4499	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4500	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4501	AssertRCReturn(rc, rc);
4502	}
4503	}
4504
4505	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase);
4506	AssertRC(rc);
4507
4508	/*
4509	* Host FS base and GS base.
4510	*/
4511	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4512	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4513	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRC(rc);
4514	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRC(rc);
4515
4516	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4517	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4518	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4519	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4520	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4521
4522	return VINF_SUCCESS;
4523	#undef VMXLOCAL_ADJUST_HOST_SEG
4524	}
4525
4526
4527	/**
4528	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4529	* host-state area of the VMCS.
4530	*
4531	* These MSRs will be automatically restored on the host after every successful
4532	* VM-exit.
4533	*
4534	* @param pVCpu The cross context virtual CPU structure.
4535	*
4536	* @remarks No-long-jump zone!!!
4537	*/
4538	static void hmR0VmxExportHostMsrs(PVMCPUCC pVCpu)
4539	{
4540	AssertPtr(pVCpu);
4541
4542	/*
4543	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4544	* rather than swapping them on every VM-entry.
4545	*/
4546	hmR0VmxLazySaveHostMsrs(pVCpu);
4547
4548	/*
4549	* Host Sysenter MSRs.
4550	*/
4551	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)); AssertRC(rc);
4552	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP)); AssertRC(rc);
4553	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP)); AssertRC(rc);
4554
4555	/*
4556	* Host EFER MSR.
4557	*
4558	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4559	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4560	*/
4561	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4562	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4563	{
4564	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4565	AssertRC(rc);
4566	}
4567
4568	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4569	* hmR0VmxExportGuestEntryExitCtls(). */
4570	}
4571
4572
4573	/**
4574	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4575	*
4576	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4577	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4578	*
4579	* @returns true if we need to load guest EFER, false otherwise.
4580	* @param pVCpu The cross context virtual CPU structure.
4581	* @param pVmxTransient The VMX-transient structure.
4582	*
4583	* @remarks Requires EFER, CR4.
4584	* @remarks No-long-jump zone!!!
4585	*/
4586	static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4587	{
4588	#ifdef HMVMX_ALWAYS_SWAP_EFER
4589	RT_NOREF2(pVCpu, pVmxTransient);
4590	return true;
4591	#else
4592	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4593	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4594	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4595	uint64_t const u64GuestEfer = pCtx->msrEFER;
4596
4597	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4598	/*
4599	* For nested-guests, we shall honor swapping the EFER MSR when requested by
4600	* the nested-guest.
4601	*/
4602	if ( pVmxTransient->fIsNestedGuest
4603	&& ( CPUMIsGuestVmxEntryCtlsSet(pVCpu, pCtx, VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4604	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_SAVE_EFER_MSR)
4605	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_LOAD_EFER_MSR)))
4606	return true;
4607	# else
4608	RT_NOREF(pVmxTransient);
4609	#endif
4610
4611	/*
4612	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4613	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4614	*/
4615	if ( CPUMIsGuestInLongModeEx(pCtx)
4616	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4617	return true;
4618
4619	/*
4620	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4621	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4622	*
4623	* See Intel spec. 4.5 "IA-32e Paging".
4624	* See Intel spec. 4.1.1 "Three Paging Modes".
4625	*
4626	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4627	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4628	*/
4629	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4630	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4631	if ( (pCtx->cr4 & X86_CR4_PAE)
4632	&& (pCtx->cr0 & X86_CR0_PG))
4633	{
4634	/*
4635	* If nested paging is not used, verify that the guest paging mode matches the
4636	* shadow paging mode which is/will be placed in the VMCS (which is what will
4637	* actually be used while executing the guest and not the CR4 shadow value).
4638	*/
4639	AssertMsg(pVM->hm.s.fNestedPaging \|\| ( pVCpu->hm.s.enmShadowMode == PGMMODE_PAE
4640	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_PAE_NX
4641	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64
4642	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64_NX),
4643	("enmShadowMode=%u\n", pVCpu->hm.s.enmShadowMode));
4644	if ((u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4645	{
4646	/* Verify that the host is NX capable. */
4647	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4648	return true;
4649	}
4650	}
4651
4652	return false;
4653	#endif
4654	}
4655
4656
4657	/**
4658	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4659	* VMCS.
4660	*
4661	* This is typically required when the guest changes paging mode.
4662	*
4663	* @returns VBox status code.
4664	* @param pVCpu The cross context virtual CPU structure.
4665	* @param pVmxTransient The VMX-transient structure.
4666	*
4667	* @remarks Requires EFER.
4668	* @remarks No-long-jump zone!!!
4669	*/
4670	static int hmR0VmxExportGuestEntryExitCtls(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4671	{
4672	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4673	{
4674	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4675	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4676
4677	/*
4678	* VM-entry controls.
4679	*/
4680	{
4681	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4682	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4683
4684	/*
4685	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4686	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4687	*
4688	* For nested-guests, this is a mandatory VM-entry control. It's also
4689	* required because we do not want to leak host bits to the nested-guest.
4690	*/
4691	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4692
4693	/*
4694	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4695	*
4696	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4697	* required to get the nested-guest working with hardware-assisted VMX execution.
4698	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
4699	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4700	* here rather than while merging the guest VMCS controls.
4701	*/
4702	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4703	{
4704	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
4705	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4706	}
4707	else
4708	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4709
4710	/*
4711	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4712	*
4713	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4714	* regardless of whether the nested-guest VMCS specifies it because we are free to
4715	* load whatever MSRs we require and we do not need to modify the guest visible copy
4716	* of the VM-entry MSR load area.
4717	*/
4718	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4719	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4720	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4721	else
4722	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4723
4724	/*
4725	* The following should -not- be set (since we're not in SMM mode):
4726	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4727	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4728	*/
4729
4730	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4731	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4732
4733	if ((fVal & fZap) == fVal)
4734	{ /* likely */ }
4735	else
4736	{
4737	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4738	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4739	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4740	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4741	}
4742
4743	/* Commit it to the VMCS. */
4744	if (pVmcsInfo->u32EntryCtls != fVal)
4745	{
4746	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4747	AssertRC(rc);
4748	pVmcsInfo->u32EntryCtls = fVal;
4749	}
4750	}
4751
4752	/*
4753	* VM-exit controls.
4754	*/
4755	{
4756	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4757	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4758
4759	/*
4760	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4761	* supported the 1-setting of this bit.
4762	*
4763	* For nested-guests, we set the "save debug controls" as the converse
4764	* "load debug controls" is mandatory for nested-guests anyway.
4765	*/
4766	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4767
4768	/*
4769	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4770	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4771	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4772	* hmR0VmxExportHostMsrs().
4773	*
4774	* For nested-guests, we always set this bit as we do not support 32-bit
4775	* hosts.
4776	*/
4777	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4778
4779	/*
4780	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4781	*
4782	* For nested-guests, we should use the "save IA32_EFER" control if we also
4783	* used the "load IA32_EFER" control while exporting VM-entry controls.
4784	*/
4785	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4786	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4787	{
4788	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4789	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4790	}
4791
4792	/*
4793	* Enable saving of the VMX-preemption timer value on VM-exit.
4794	* For nested-guests, currently not exposed/used.
4795	*/
4796	if ( pVM->hm.s.vmx.fUsePreemptTimer
4797	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4798	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4799
4800	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4801	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4802
4803	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4804	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4805	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4806
4807	if ((fVal & fZap) == fVal)
4808	{ /* likely */ }
4809	else
4810	{
4811	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4812	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4813	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4814	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4815	}
4816
4817	/* Commit it to the VMCS. */
4818	if (pVmcsInfo->u32ExitCtls != fVal)
4819	{
4820	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4821	AssertRC(rc);
4822	pVmcsInfo->u32ExitCtls = fVal;
4823	}
4824	}
4825
4826	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4827	}
4828	return VINF_SUCCESS;
4829	}
4830
4831
4832	/**
4833	* Sets the TPR threshold in the VMCS.
4834	*
4835	* @param pVmcsInfo The VMCS info. object.
4836	* @param u32TprThreshold The TPR threshold (task-priority class only).
4837	*/
4838	DECLINLINE(void) hmR0VmxApicSetTprThreshold(PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4839	{
4840	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4841	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4842	RT_NOREF(pVmcsInfo);
4843	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4844	AssertRC(rc);
4845	}
4846
4847
4848	/**
4849	* Exports the guest APIC TPR state into the VMCS.
4850	*
4851	* @param pVCpu The cross context virtual CPU structure.
4852	* @param pVmxTransient The VMX-transient structure.
4853	*
4854	* @remarks No-long-jump zone!!!
4855	*/
4856	static void hmR0VmxExportGuestApicTpr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4857	{
4858	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4859	{
4860	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4861
4862	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4863	if (!pVmxTransient->fIsNestedGuest)
4864	{
4865	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4866	&& APICIsEnabled(pVCpu))
4867	{
4868	/*
4869	* Setup TPR shadowing.
4870	*/
4871	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4872	{
4873	bool fPendingIntr = false;
4874	uint8_t u8Tpr = 0;
4875	uint8_t u8PendingIntr = 0;
4876	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4877	AssertRC(rc);
4878
4879	/*
4880	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4881	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4882	* priority of the pending interrupt so we can deliver the interrupt. If there
4883	* are no interrupts pending, set threshold to 0 to not cause any
4884	* TPR-below-threshold VM-exits.
4885	*/
4886	uint32_t u32TprThreshold = 0;
4887	if (fPendingIntr)
4888	{
4889	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4890	(which is the Task-Priority Class). */
4891	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4892	const uint8_t u8TprPriority = u8Tpr >> 4;
4893	if (u8PendingPriority <= u8TprPriority)
4894	u32TprThreshold = u8PendingPriority;
4895	}
4896
4897	hmR0VmxApicSetTprThreshold(pVmcsInfo, u32TprThreshold);
4898	}
4899	}
4900	}
4901	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4902	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4903	}
4904	}
4905
4906
4907	/**
4908	* Gets the guest interruptibility-state.
4909	*
4910	* @returns Guest's interruptibility-state.
4911	* @param pVCpu The cross context virtual CPU structure.
4912	* @param pVmxTransient The VMX-transient structure.
4913	*
4914	* @remarks No-long-jump zone!!!
4915	*/
4916	static uint32_t hmR0VmxGetGuestIntrState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4917	{
4918	/*
4919	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4920	*/
4921	uint32_t fIntrState = 0;
4922	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4923	{
4924	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
4925	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
4926
4927	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4928	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4929	{
4930	if (pCtx->eflags.Bits.u1IF)
4931	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4932	else
4933	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4934	}
4935	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4936	{
4937	/*
4938	* We can clear the inhibit force flag as even if we go back to the recompiler
4939	* without executing guest code in VT-x, the flag's condition to be cleared is
4940	* met and thus the cleared state is correct.
4941	*/
4942	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4943	}
4944	}
4945
4946	/*
4947	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4948	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4949	* setting this would block host-NMIs and IRET will not clear the blocking.
4950	*
4951	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4952	*
4953	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4954	*/
4955	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4956	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4957	&& CPUMIsGuestNmiBlocking(pVCpu))
4958	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4959
4960	return fIntrState;
4961	}
4962
4963
4964	/**
4965	* Exports the exception intercepts required for guest execution in the VMCS.
4966	*
4967	* @param pVCpu The cross context virtual CPU structure.
4968	* @param pVmxTransient The VMX-transient structure.
4969	*
4970	* @remarks No-long-jump zone!!!
4971	*/
4972	static void hmR0VmxExportGuestXcptIntercepts(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4973	{
4974	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4975	{
4976	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4977	if ( !pVmxTransient->fIsNestedGuest
4978	&& pVCpu->hm.s.fGIMTrapXcptUD)
4979	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4980	else
4981	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4982
4983	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4984	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4985	}
4986	}
4987
4988
4989	/**
4990	* Exports the guest's RIP into the guest-state area in the VMCS.
4991	*
4992	* @param pVCpu The cross context virtual CPU structure.
4993	*
4994	* @remarks No-long-jump zone!!!
4995	*/
4996	static void hmR0VmxExportGuestRip(PVMCPUCC pVCpu)
4997	{
4998	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4999	{
5000	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5001
5002	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5003	AssertRC(rc);
5004
5005	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5006	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5007	}
5008	}
5009
5010
5011	/**
5012	* Exports the guest's RSP into the guest-state area in the VMCS.
5013	*
5014	* @param pVCpu The cross context virtual CPU structure.
5015	*
5016	* @remarks No-long-jump zone!!!
5017	*/
5018	static void hmR0VmxExportGuestRsp(PVMCPUCC pVCpu)
5019	{
5020	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5021	{
5022	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5023
5024	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5025	AssertRC(rc);
5026
5027	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5028	Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
5029	}
5030	}
5031
5032
5033	/**
5034	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5035	*
5036	* @param pVCpu The cross context virtual CPU structure.
5037	* @param pVmxTransient The VMX-transient structure.
5038	*
5039	* @remarks No-long-jump zone!!!
5040	*/
5041	static void hmR0VmxExportGuestRflags(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5042	{
5043	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5044	{
5045	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5046
5047	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5048	Let us assert it as such and use 32-bit VMWRITE. */
5049	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5050	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5051	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5052	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5053
5054	/*
5055	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5056	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5057	* can run the real-mode guest code under Virtual 8086 mode.
5058	*/
5059	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5060	if (pVmcsInfo->RealMode.fRealOnV86Active)
5061	{
5062	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5063	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5064	Assert(!pVmxTransient->fIsNestedGuest);
5065	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5066	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5067	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5068	}
5069
5070	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5071	AssertRC(rc);
5072
5073	/*
5074	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5075	*
5076	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5077	* through the hypervisor debugger using EFLAGS.TF.
5078	*/
5079	if ( !pVmxTransient->fIsNestedGuest
5080	&& !pVCpu->hm.s.fSingleInstruction
5081	&& fEFlags.Bits.u1TF)
5082	{
5083	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5084	* premature trips to ring-3 esp since IEM does not yet handle it. */
5085	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5086	AssertRC(rc);
5087	}
5088	/* else: for nested-guest currently handling while merging controls. */
5089
5090	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5091	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5092	}
5093	}
5094
5095
5096	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5097	/**
5098	* Copies the nested-guest VMCS to the shadow VMCS.
5099	*
5100	* @returns VBox status code.
5101	* @param pVCpu The cross context virtual CPU structure.
5102	* @param pVmcsInfo The VMCS info. object.
5103	*
5104	* @remarks No-long-jump zone!!!
5105	*/
5106	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5107	{
5108	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5109	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5110
5111	/*
5112	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5113	* current VMCS, as we may try saving guest lazy MSRs.
5114	*
5115	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5116	* calling the import VMCS code which is currently performing the guest MSR reads
5117	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5118	* and the rest of the VMX leave session machinery.
5119	*/
5120	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5121
5122	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5123	if (RT_SUCCESS(rc))
5124	{
5125	/*
5126	* Copy all guest read/write VMCS fields.
5127	*
5128	* We don't check for VMWRITE failures here for performance reasons and
5129	* because they are not expected to fail, barring irrecoverable conditions
5130	* like hardware errors.
5131	*/
5132	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5133	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5134	{
5135	uint64_t u64Val;
5136	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5137	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5138	VMXWriteVmcs64(uVmcsField, u64Val);
5139	}
5140
5141	/*
5142	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5143	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5144	*/
5145	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5146	{
5147	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5148	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5149	{
5150	uint64_t u64Val;
5151	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5152	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5153	VMXWriteVmcs64(uVmcsField, u64Val);
5154	}
5155	}
5156
5157	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5158	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5159	}
5160
5161	ASMSetFlags(fEFlags);
5162	return rc;
5163	}
5164
5165
5166	/**
5167	* Copies the shadow VMCS to the nested-guest VMCS.
5168	*
5169	* @returns VBox status code.
5170	* @param pVCpu The cross context virtual CPU structure.
5171	* @param pVmcsInfo The VMCS info. object.
5172	*
5173	* @remarks Called with interrupts disabled.
5174	*/
5175	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5176	{
5177	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5178	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5179	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5180
5181	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5182	if (RT_SUCCESS(rc))
5183	{
5184	/*
5185	* Copy guest read/write fields from the shadow VMCS.
5186	* Guest read-only fields cannot be modified, so no need to copy them.
5187	*
5188	* We don't check for VMREAD failures here for performance reasons and
5189	* because they are not expected to fail, barring irrecoverable conditions
5190	* like hardware errors.
5191	*/
5192	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5193	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5194	{
5195	uint64_t u64Val;
5196	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5197	VMXReadVmcs64(uVmcsField, &u64Val);
5198	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5199	}
5200
5201	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5202	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5203	}
5204	return rc;
5205	}
5206
5207
5208	/**
5209	* Enables VMCS shadowing for the given VMCS info. object.
5210	*
5211	* @param pVmcsInfo The VMCS info. object.
5212	*
5213	* @remarks No-long-jump zone!!!
5214	*/
5215	static void hmR0VmxEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5216	{
5217	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5218	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5219	{
5220	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5221	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5222	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5223	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
5224	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5225	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5226	Log4Func(("Enabled\n"));
5227	}
5228	}
5229
5230
5231	/**
5232	* Disables VMCS shadowing for the given VMCS info. object.
5233	*
5234	* @param pVmcsInfo The VMCS info. object.
5235	*
5236	* @remarks No-long-jump zone!!!
5237	*/
5238	static void hmR0VmxDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5239	{
5240	/*
5241	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5242	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5243	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5244	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5245	*
5246	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5247	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5248	*/
5249	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5250	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5251	{
5252	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5253	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5254	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
5255	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5256	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5257	Log4Func(("Disabled\n"));
5258	}
5259	}
5260	#endif
5261
5262
5263	/**
5264	* Exports the guest hardware-virtualization state.
5265	*
5266	* @returns VBox status code.
5267	* @param pVCpu The cross context virtual CPU structure.
5268	* @param pVmxTransient The VMX-transient structure.
5269	*
5270	* @remarks No-long-jump zone!!!
5271	*/
5272	static int hmR0VmxExportGuestHwvirtState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5273	{
5274	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5275	{
5276	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5277	/*
5278	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5279	* VMCS shadowing.
5280	*/
5281	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5282	{
5283	/*
5284	* If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
5285	* copy the nested hypervisor's current VMCS into the shadow VMCS and enable
5286	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5287	*
5288	* We check for VMX root mode here in case the guest executes VMXOFF without
5289	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5290	* not clear the current VMCS pointer.
5291	*/
5292	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5293	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5294	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5295	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5296	{
5297	/* Paranoia. */
5298	Assert(!pVmxTransient->fIsNestedGuest);
5299
5300	/*
5301	* For performance reasons, also check if the nested hypervisor's current VMCS
5302	* was newly loaded or modified before copying it to the shadow VMCS.
5303	*/
5304	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5305	{
5306	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5307	AssertRCReturn(rc, rc);
5308	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5309	}
5310	hmR0VmxEnableVmcsShadowing(pVmcsInfo);
5311	}
5312	else
5313	hmR0VmxDisableVmcsShadowing(pVmcsInfo);
5314	}
5315	#else
5316	NOREF(pVmxTransient);
5317	#endif
5318	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5319	}
5320	return VINF_SUCCESS;
5321	}
5322
5323
5324	/**
5325	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5326	*
5327	* The guest FPU state is always pre-loaded hence we don't need to bother about
5328	* sharing FPU related CR0 bits between the guest and host.
5329	*
5330	* @returns VBox status code.
5331	* @param pVCpu The cross context virtual CPU structure.
5332	* @param pVmxTransient The VMX-transient structure.
5333	*
5334	* @remarks No-long-jump zone!!!
5335	*/
5336	static int hmR0VmxExportGuestCR0(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5337	{
5338	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5339	{
5340	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5341	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5342
5343	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0;
5344	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5345	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5346	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5347	else
5348	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5349
5350	if (!pVmxTransient->fIsNestedGuest)
5351	{
5352	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5353	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5354	uint64_t const u64ShadowCr0 = u64GuestCr0;
5355	Assert(!RT_HI_U32(u64GuestCr0));
5356
5357	/*
5358	* Setup VT-x's view of the guest CR0.
5359	*/
5360	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5361	if (pVM->hm.s.fNestedPaging)
5362	{
5363	if (CPUMIsGuestPagingEnabled(pVCpu))
5364	{
5365	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5366	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5367	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5368	}
5369	else
5370	{
5371	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5372	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5373	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5374	}
5375
5376	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5377	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5378	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5379	}
5380	else
5381	{
5382	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5383	u64GuestCr0 \|= X86_CR0_WP;
5384	}
5385
5386	/*
5387	* Guest FPU bits.
5388	*
5389	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5390	* using CR0.TS.
5391	*
5392	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5393	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5394	*/
5395	u64GuestCr0 \|= X86_CR0_NE;
5396
5397	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5398	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5399
5400	/*
5401	* Update exception intercepts.
5402	*/
5403	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5404	if (pVmcsInfo->RealMode.fRealOnV86Active)
5405	{
5406	Assert(PDMVmmDevHeapIsEnabled(pVM));
5407	Assert(pVM->hm.s.vmx.pRealModeTSS);
5408	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5409	}
5410	else
5411	{
5412	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5413	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5414	if (fInterceptMF)
5415	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5416	}
5417
5418	/* Additional intercepts for debugging, define these yourself explicitly. */
5419	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5420	uXcptBitmap \|= 0
5421	\| RT_BIT(X86_XCPT_BP)
5422	\| RT_BIT(X86_XCPT_DE)
5423	\| RT_BIT(X86_XCPT_NM)
5424	\| RT_BIT(X86_XCPT_TS)
5425	\| RT_BIT(X86_XCPT_UD)
5426	\| RT_BIT(X86_XCPT_NP)
5427	\| RT_BIT(X86_XCPT_SS)
5428	\| RT_BIT(X86_XCPT_GP)
5429	\| RT_BIT(X86_XCPT_PF)
5430	\| RT_BIT(X86_XCPT_MF)
5431	;
5432	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5433	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5434	#endif
5435	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5436	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5437	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5438
5439	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5440	u64GuestCr0 \|= fSetCr0;
5441	u64GuestCr0 &= fZapCr0;
5442	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5443
5444	/* Commit the CR0 and related fields to the guest VMCS. */
5445	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5446	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5447	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5448	{
5449	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5450	AssertRC(rc);
5451	}
5452	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5453	{
5454	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5455	AssertRC(rc);
5456	}
5457
5458	/* Update our caches. */
5459	pVmcsInfo->u32ProcCtls = uProcCtls;
5460	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5461
5462	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5463	}
5464	else
5465	{
5466	/*
5467	* With nested-guests, we may have extended the guest/host mask here since we
5468	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5469	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
5470	* originally supplied. We must copy those bits from the nested-guest CR0 into
5471	* the nested-guest CR0 read-shadow.
5472	*/
5473	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5474	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5475	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5476	Assert(!RT_HI_U32(u64GuestCr0));
5477	Assert(u64GuestCr0 & X86_CR0_NE);
5478
5479	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5480	u64GuestCr0 \|= fSetCr0;
5481	u64GuestCr0 &= fZapCr0;
5482	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5483
5484	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5485	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5486	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5487
5488	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5489	}
5490
5491	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5492	}
5493
5494	return VINF_SUCCESS;
5495	}
5496
5497
5498	/**
5499	* Exports the guest control registers (CR3, CR4) into the guest-state area
5500	* in the VMCS.
5501	*
5502	* @returns VBox strict status code.
5503	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5504	* without unrestricted guest access and the VMMDev is not presently
5505	* mapped (e.g. EFI32).
5506	*
5507	* @param pVCpu The cross context virtual CPU structure.
5508	* @param pVmxTransient The VMX-transient structure.
5509	*
5510	* @remarks No-long-jump zone!!!
5511	*/
5512	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5513	{
5514	int rc = VINF_SUCCESS;
5515	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5516
5517	/*
5518	* Guest CR2.
5519	* It's always loaded in the assembler code. Nothing to do here.
5520	*/
5521
5522	/*
5523	* Guest CR3.
5524	*/
5525	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5526	{
5527	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5528
5529	if (pVM->hm.s.fNestedPaging)
5530	{
5531	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5532	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5533
5534	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5535	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5536	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5537	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5538
5539	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5540	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5541	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5542
5543	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5544	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5545	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5546	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5547	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5548	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5549	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5550
5551	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5552	AssertRC(rc);
5553
5554	uint64_t u64GuestCr3;
5555	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5556	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5557	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5558	{
5559	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5560	if (CPUMIsGuestInPAEModeEx(pCtx))
5561	{
5562	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5563	AssertRC(rc);
5564	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
5565	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
5566	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
5567	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
5568	}
5569
5570	/*
5571	* The guest's view of its CR3 is unblemished with nested paging when the
5572	* guest is using paging or we have unrestricted guest execution to handle
5573	* the guest when it's not using paging.
5574	*/
5575	u64GuestCr3 = pCtx->cr3;
5576	}
5577	else
5578	{
5579	/*
5580	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5581	* thinks it accesses physical memory directly, we use our identity-mapped
5582	* page table to map guest-linear to guest-physical addresses. EPT takes care
5583	* of translating it to host-physical addresses.
5584	*/
5585	RTGCPHYS GCPhys;
5586	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5587
5588	/* We obtain it here every time as the guest could have relocated this PCI region. */
5589	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5590	if (RT_SUCCESS(rc))
5591	{ /* likely */ }
5592	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5593	{
5594	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5595	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5596	}
5597	else
5598	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5599
5600	u64GuestCr3 = GCPhys;
5601	}
5602
5603	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
5604	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, u64GuestCr3);
5605	AssertRC(rc);
5606	}
5607	else
5608	{
5609	Assert(!pVmxTransient->fIsNestedGuest);
5610	/* Non-nested paging case, just use the hypervisor's CR3. */
5611	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5612
5613	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
5614	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5615	AssertRC(rc);
5616	}
5617
5618	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5619	}
5620
5621	/*
5622	* Guest CR4.
5623	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5624	*/
5625	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5626	{
5627	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5628	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5629
5630	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0;
5631	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5632
5633	/*
5634	* With nested-guests, we may have extended the guest/host mask here (since we
5635	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5636	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5637	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
5638	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5639	*/
5640	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5641	uint64_t u64GuestCr4 = pCtx->cr4;
5642	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5643	? pCtx->cr4
5644	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5645	Assert(!RT_HI_U32(u64GuestCr4));
5646
5647	/*
5648	* Setup VT-x's view of the guest CR4.
5649	*
5650	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5651	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5652	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5653	*
5654	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5655	*/
5656	if (pVmcsInfo->RealMode.fRealOnV86Active)
5657	{
5658	Assert(pVM->hm.s.vmx.pRealModeTSS);
5659	Assert(PDMVmmDevHeapIsEnabled(pVM));
5660	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5661	}
5662
5663	if (pVM->hm.s.fNestedPaging)
5664	{
5665	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5666	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5667	{
5668	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5669	u64GuestCr4 \|= X86_CR4_PSE;
5670	/* Our identity mapping is a 32-bit page directory. */
5671	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5672	}
5673	/* else use guest CR4.*/
5674	}
5675	else
5676	{
5677	Assert(!pVmxTransient->fIsNestedGuest);
5678
5679	/*
5680	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5681	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5682	*/
5683	switch (pVCpu->hm.s.enmShadowMode)
5684	{
5685	case PGMMODE_REAL: /* Real-mode. */
5686	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5687	case PGMMODE_32_BIT: /* 32-bit paging. */
5688	{
5689	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5690	break;
5691	}
5692
5693	case PGMMODE_PAE: /* PAE paging. */
5694	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5695	{
5696	u64GuestCr4 \|= X86_CR4_PAE;
5697	break;
5698	}
5699
5700	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5701	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5702	{
5703	#ifdef VBOX_WITH_64_BITS_GUESTS
5704	/* For our assumption in hmR0VmxShouldSwapEferMsr. */
5705	Assert(u64GuestCr4 & X86_CR4_PAE);
5706	break;
5707	#endif
5708	}
5709	default:
5710	AssertFailed();
5711	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5712	}
5713	}
5714
5715	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
5716	u64GuestCr4 \|= fSetCr4;
5717	u64GuestCr4 &= fZapCr4;
5718
5719	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5720	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
5721	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
5722
5723	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5724	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5725
5726	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5727
5728	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5729	}
5730	return rc;
5731	}
5732
5733
5734	/**
5735	* Exports the guest debug registers into the guest-state area in the VMCS.
5736	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5737	*
5738	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5739	*
5740	* @returns VBox status code.
5741	* @param pVCpu The cross context virtual CPU structure.
5742	* @param pVmxTransient The VMX-transient structure.
5743	*
5744	* @remarks No-long-jump zone!!!
5745	*/
5746	static int hmR0VmxExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5747	{
5748	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5749
5750	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5751	* stepping. */
5752	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5753	if (pVmxTransient->fIsNestedGuest)
5754	{
5755	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5756	AssertRC(rc);
5757
5758	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5759	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5760	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5761	AssertRC(rc);
5762	return VINF_SUCCESS;
5763	}
5764
5765	#ifdef VBOX_STRICT
5766	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5767	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5768	{
5769	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5770	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5771	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5772	}
5773	#endif
5774
5775	bool fSteppingDB = false;
5776	bool fInterceptMovDRx = false;
5777	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5778	if (pVCpu->hm.s.fSingleInstruction)
5779	{
5780	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5781	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5782	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5783	{
5784	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5785	Assert(fSteppingDB == false);
5786	}
5787	else
5788	{
5789	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5790	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5791	pVCpu->hm.s.fClearTrapFlag = true;
5792	fSteppingDB = true;
5793	}
5794	}
5795
5796	uint64_t u64GuestDr7;
5797	if ( fSteppingDB
5798	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5799	{
5800	/*
5801	* Use the combined guest and host DRx values found in the hypervisor register set
5802	* because the hypervisor debugger has breakpoints active or someone is single stepping
5803	* on the host side without a monitor trap flag.
5804	*
5805	* Note! DBGF expects a clean DR6 state before executing guest code.
5806	*/
5807	if (!CPUMIsHyperDebugStateActive(pVCpu))
5808	{
5809	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5810	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5811	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5812	}
5813
5814	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5815	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
5816	pVCpu->hm.s.fUsingHyperDR7 = true;
5817	fInterceptMovDRx = true;
5818	}
5819	else
5820	{
5821	/*
5822	* If the guest has enabled debug registers, we need to load them prior to
5823	* executing guest code so they'll trigger at the right time.
5824	*/
5825	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5826	{
5827	if (!CPUMIsGuestDebugStateActive(pVCpu))
5828	{
5829	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5830	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5831	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5832	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5833	}
5834	Assert(!fInterceptMovDRx);
5835	}
5836	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5837	{
5838	/*
5839	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5840	* must intercept #DB in order to maintain a correct DR6 guest value, and
5841	* because we need to intercept it to prevent nested #DBs from hanging the
5842	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5843	*/
5844	fInterceptMovDRx = true;
5845	}
5846
5847	/* Update DR7 with the actual guest value. */
5848	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5849	pVCpu->hm.s.fUsingHyperDR7 = false;
5850	}
5851
5852	if (fInterceptMovDRx)
5853	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5854	else
5855	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5856
5857	/*
5858	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5859	* monitor-trap flag and update our cache.
5860	*/
5861	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5862	{
5863	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5864	AssertRC(rc);
5865	pVmcsInfo->u32ProcCtls = uProcCtls;
5866	}
5867
5868	/*
5869	* Update guest DR7.
5870	*/
5871	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
5872	AssertRC(rc);
5873
5874	/*
5875	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5876	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5877	*
5878	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5879	*/
5880	if (fSteppingDB)
5881	{
5882	Assert(pVCpu->hm.s.fSingleInstruction);
5883	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5884
5885	uint32_t fIntrState = 0;
5886	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5887	AssertRC(rc);
5888
5889	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5890	{
5891	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5892	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5893	AssertRC(rc);
5894	}
5895	}
5896
5897	return VINF_SUCCESS;
5898	}
5899
5900
5901	#ifdef VBOX_STRICT
5902	/**
5903	* Strict function to validate segment registers.
5904	*
5905	* @param pVCpu The cross context virtual CPU structure.
5906	* @param pVmcsInfo The VMCS info. object.
5907	*
5908	* @remarks Will import guest CR0 on strict builds during validation of
5909	* segments.
5910	*/
5911	static void hmR0VmxValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5912	{
5913	/*
5914	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5915	*
5916	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5917	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5918	* unusable bit and doesn't change the guest-context value.
5919	*/
5920	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5921	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5922	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5923	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5924	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5925	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5926	{
5927	/* Protected mode checks */
5928	/* CS */
5929	Assert(pCtx->cs.Attr.n.u1Present);
5930	Assert(!(pCtx->cs.Attr.u & 0xf00));
5931	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5932	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5933	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5934	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5935	\|\| (pCtx->cs.Attr.n.u1Granularity));
5936	/* CS cannot be loaded with NULL in protected mode. */
5937	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5938	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5939	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5940	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5941	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5942	else
5943	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5944	/* SS */
5945	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5946	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5947	if ( !(pCtx->cr0 & X86_CR0_PE)
5948	\|\| pCtx->cs.Attr.n.u4Type == 3)
5949	{
5950	Assert(!pCtx->ss.Attr.n.u2Dpl);
5951	}
5952	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5953	{
5954	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5955	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5956	Assert(pCtx->ss.Attr.n.u1Present);
5957	Assert(!(pCtx->ss.Attr.u & 0xf00));
5958	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5959	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5960	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5961	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5962	\|\| (pCtx->ss.Attr.n.u1Granularity));
5963	}
5964	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5965	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5966	{
5967	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5968	Assert(pCtx->ds.Attr.n.u1Present);
5969	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5970	Assert(!(pCtx->ds.Attr.u & 0xf00));
5971	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5972	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5973	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5974	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5975	\|\| (pCtx->ds.Attr.n.u1Granularity));
5976	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5977	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5978	}
5979	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5980	{
5981	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5982	Assert(pCtx->es.Attr.n.u1Present);
5983	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5984	Assert(!(pCtx->es.Attr.u & 0xf00));
5985	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5986	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5987	\|\| !(pCtx->es.Attr.n.u1Granularity));
5988	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5989	\|\| (pCtx->es.Attr.n.u1Granularity));
5990	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5991	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5992	}
5993	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5994	{
5995	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5996	Assert(pCtx->fs.Attr.n.u1Present);
5997	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5998	Assert(!(pCtx->fs.Attr.u & 0xf00));
5999	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6000	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6001	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6002	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6003	\|\| (pCtx->fs.Attr.n.u1Granularity));
6004	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6005	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6006	}
6007	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6008	{
6009	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6010	Assert(pCtx->gs.Attr.n.u1Present);
6011	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6012	Assert(!(pCtx->gs.Attr.u & 0xf00));
6013	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6014	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6015	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6016	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6017	\|\| (pCtx->gs.Attr.n.u1Granularity));
6018	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6019	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6020	}
6021	/* 64-bit capable CPUs. */
6022	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6023	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6024	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6025	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6026	}
6027	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6028	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6029	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6030	{
6031	/* Real and v86 mode checks. */
6032	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6033	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6034	if (pVmcsInfo->RealMode.fRealOnV86Active)
6035	{
6036	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6037	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6038	}
6039	else
6040	{
6041	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6042	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6043	}
6044
6045	/* CS */
6046	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6047	Assert(pCtx->cs.u32Limit == 0xffff);
6048	Assert(u32CSAttr == 0xf3);
6049	/* SS */
6050	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6051	Assert(pCtx->ss.u32Limit == 0xffff);
6052	Assert(u32SSAttr == 0xf3);
6053	/* DS */
6054	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6055	Assert(pCtx->ds.u32Limit == 0xffff);
6056	Assert(u32DSAttr == 0xf3);
6057	/* ES */
6058	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6059	Assert(pCtx->es.u32Limit == 0xffff);
6060	Assert(u32ESAttr == 0xf3);
6061	/* FS */
6062	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6063	Assert(pCtx->fs.u32Limit == 0xffff);
6064	Assert(u32FSAttr == 0xf3);
6065	/* GS */
6066	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6067	Assert(pCtx->gs.u32Limit == 0xffff);
6068	Assert(u32GSAttr == 0xf3);
6069	/* 64-bit capable CPUs. */
6070	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6071	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6072	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6073	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6074	}
6075	}
6076	#endif /* VBOX_STRICT */
6077
6078
6079	/**
6080	* Exports a guest segment register into the guest-state area in the VMCS.
6081	*
6082	* @returns VBox status code.
6083	* @param pVCpu The cross context virtual CPU structure.
6084	* @param pVmcsInfo The VMCS info. object.
6085	* @param iSegReg The segment register number (X86_SREG_XXX).
6086	* @param pSelReg Pointer to the segment selector.
6087	*
6088	* @remarks No-long-jump zone!!!
6089	*/
6090	static int hmR0VmxExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6091	{
6092	Assert(iSegReg < X86_SREG_COUNT);
6093	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6094	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6095	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6096	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6097
6098	uint32_t u32Access = pSelReg->Attr.u;
6099	if (pVmcsInfo->RealMode.fRealOnV86Active)
6100	{
6101	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6102	u32Access = 0xf3;
6103	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6104	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6105	RT_NOREF_PV(pVCpu);
6106	}
6107	else
6108	{
6109	/*
6110	* The way to differentiate between whether this is really a null selector or was just
6111	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6112	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6113	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6114	* NULL selectors loaded in protected-mode have their attribute as 0.
6115	*/
6116	if (!u32Access)
6117	u32Access = X86DESCATTR_UNUSABLE;
6118	}
6119
6120	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6121	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6122	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6123
6124	/*
6125	* Commit it to the VMCS.
6126	*/
6127	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); AssertRC(rc);
6128	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); AssertRC(rc);
6129	rc = VMXWriteVmcsNw(idxBase, pSelReg->u64Base); AssertRC(rc);
6130	rc = VMXWriteVmcs32(idxAttr, u32Access); AssertRC(rc);
6131	return VINF_SUCCESS;
6132	}
6133
6134
6135	/**
6136	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6137	* area in the VMCS.
6138	*
6139	* @returns VBox status code.
6140	* @param pVCpu The cross context virtual CPU structure.
6141	* @param pVmxTransient The VMX-transient structure.
6142	*
6143	* @remarks Will import guest CR0 on strict builds during validation of
6144	* segments.
6145	* @remarks No-long-jump zone!!!
6146	*/
6147	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6148	{
6149	int rc = VERR_INTERNAL_ERROR_5;
6150	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6151	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6152	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6153
6154	/*
6155	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6156	*/
6157	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6158	{
6159	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6160	{
6161	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6162	if (pVmcsInfo->RealMode.fRealOnV86Active)
6163	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6164	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6165	AssertRC(rc);
6166	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6167	}
6168
6169	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6170	{
6171	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6172	if (pVmcsInfo->RealMode.fRealOnV86Active)
6173	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6174	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6175	AssertRC(rc);
6176	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6177	}
6178
6179	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6180	{
6181	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6182	if (pVmcsInfo->RealMode.fRealOnV86Active)
6183	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6184	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6185	AssertRC(rc);
6186	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6187	}
6188
6189	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6190	{
6191	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6192	if (pVmcsInfo->RealMode.fRealOnV86Active)
6193	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6194	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6195	AssertRC(rc);
6196	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6197	}
6198
6199	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6200	{
6201	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6202	if (pVmcsInfo->RealMode.fRealOnV86Active)
6203	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6204	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6205	AssertRC(rc);
6206	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6207	}
6208
6209	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6210	{
6211	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6212	if (pVmcsInfo->RealMode.fRealOnV86Active)
6213	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6214	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6215	AssertRC(rc);
6216	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6217	}
6218
6219	#ifdef VBOX_STRICT
6220	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6221	#endif
6222	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6223	pCtx->cs.Attr.u));
6224	}
6225
6226	/*
6227	* Guest TR.
6228	*/
6229	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6230	{
6231	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6232
6233	/*
6234	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6235	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6236	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6237	*/
6238	uint16_t u16Sel;
6239	uint32_t u32Limit;
6240	uint64_t u64Base;
6241	uint32_t u32AccessRights;
6242	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6243	{
6244	u16Sel = pCtx->tr.Sel;
6245	u32Limit = pCtx->tr.u32Limit;
6246	u64Base = pCtx->tr.u64Base;
6247	u32AccessRights = pCtx->tr.Attr.u;
6248	}
6249	else
6250	{
6251	Assert(!pVmxTransient->fIsNestedGuest);
6252	Assert(pVM->hm.s.vmx.pRealModeTSS);
6253	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6254
6255	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6256	RTGCPHYS GCPhys;
6257	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6258	AssertRCReturn(rc, rc);
6259
6260	X86DESCATTR DescAttr;
6261	DescAttr.u = 0;
6262	DescAttr.n.u1Present = 1;
6263	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6264
6265	u16Sel = 0;
6266	u32Limit = HM_VTX_TSS_SIZE;
6267	u64Base = GCPhys;
6268	u32AccessRights = DescAttr.u;
6269	}
6270
6271	/* Validate. */
6272	Assert(!(u16Sel & RT_BIT(2)));
6273	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6274	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6275	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6276	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6277	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6278	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6279	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6280	Assert( (u32Limit & 0xfff) == 0xfff
6281	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6282	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6283	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6284
6285	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
6286	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
6287	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
6288	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
6289
6290	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6291	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6292	}
6293
6294	/*
6295	* Guest GDTR.
6296	*/
6297	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6298	{
6299	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6300
6301	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
6302	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
6303
6304	/* Validate. */
6305	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6306
6307	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6308	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6309	}
6310
6311	/*
6312	* Guest LDTR.
6313	*/
6314	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6315	{
6316	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6317
6318	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6319	uint32_t u32Access;
6320	if ( !pVmxTransient->fIsNestedGuest
6321	&& !pCtx->ldtr.Attr.u)
6322	u32Access = X86DESCATTR_UNUSABLE;
6323	else
6324	u32Access = pCtx->ldtr.Attr.u;
6325
6326	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
6327	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
6328	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
6329	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
6330
6331	/* Validate. */
6332	if (!(u32Access & X86DESCATTR_UNUSABLE))
6333	{
6334	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6335	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6336	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6337	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6338	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6339	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6340	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6341	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6342	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6343	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6344	}
6345
6346	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6347	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6348	}
6349
6350	/*
6351	* Guest IDTR.
6352	*/
6353	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6354	{
6355	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6356
6357	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
6358	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
6359
6360	/* Validate. */
6361	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6362
6363	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6364	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6365	}
6366
6367	return VINF_SUCCESS;
6368	}
6369
6370
6371	/**
6372	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6373	* areas.
6374	*
6375	* These MSRs will automatically be loaded to the host CPU on every successful
6376	* VM-entry and stored from the host CPU on every successful VM-exit.
6377	*
6378	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6379	* actual host MSR values are not- updated here for performance reasons. See
6380	* hmR0VmxExportHostMsrs().
6381	*
6382	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6383	*
6384	* @returns VBox status code.
6385	* @param pVCpu The cross context virtual CPU structure.
6386	* @param pVmxTransient The VMX-transient structure.
6387	*
6388	* @remarks No-long-jump zone!!!
6389	*/
6390	static int hmR0VmxExportGuestMsrs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6391	{
6392	AssertPtr(pVCpu);
6393	AssertPtr(pVmxTransient);
6394
6395	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6396	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6397
6398	/*
6399	* MSRs that we use the auto-load/store MSR area in the VMCS.
6400	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
6401	* nothing to do here. The host MSR values are updated when it's safe in
6402	* hmR0VmxLazySaveHostMsrs().
6403	*
6404	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6405	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6406	* emulation. The merged MSR permission bitmap will ensure that we get VM-exits
6407	* for any MSR that are not part of the lazy MSRs so we do not need to place
6408	* those MSRs into the auto-load/store MSR area. Nothing to do here.
6409	*/
6410	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6411	{
6412	/* No auto-load/store MSRs currently. */
6413	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6414	}
6415
6416	/*
6417	* Guest Sysenter MSRs.
6418	*/
6419	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6420	{
6421	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6422
6423	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6424	{
6425	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6426	AssertRC(rc);
6427	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6428	}
6429
6430	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6431	{
6432	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6433	AssertRC(rc);
6434	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6435	}
6436
6437	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6438	{
6439	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6440	AssertRC(rc);
6441	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6442	}
6443	}
6444
6445	/*
6446	* Guest/host EFER MSR.
6447	*/
6448	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6449	{
6450	/* Whether we are using the VMCS to swap the EFER MSR must have been
6451	determined earlier while exporting VM-entry/VM-exit controls. */
6452	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6453	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6454
6455	if (hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
6456	{
6457	/*
6458	* EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
6459	* This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
6460	* VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
6461	* and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
6462	* must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
6463	* the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
6464	* during VM-entry.
6465	*/
6466	uint64_t uGuestEferMsr = pCtx->msrEFER;
6467	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6468	{
6469	if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
6470	uGuestEferMsr &= ~MSR_K6_EFER_LME;
6471	else
6472	Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
6473	}
6474
6475	/*
6476	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6477	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6478	*/
6479	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6480	{
6481	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
6482	AssertRC(rc);
6483	}
6484	else
6485	{
6486	/*
6487	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6488	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6489	*/
6490	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
6491	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6492	AssertRCReturn(rc, rc);
6493	}
6494
6495	Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
6496	}
6497	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6498	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6499
6500	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6501	}
6502
6503	/*
6504	* Other MSRs.
6505	* Speculation Control (R/W).
6506	*/
6507	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6508	{
6509	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6510	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6511	{
6512	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6513	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6514	AssertRCReturn(rc, rc);
6515	}
6516	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6517	}
6518
6519	return VINF_SUCCESS;
6520	}
6521
6522
6523	/**
6524	* Selects up the appropriate function to run guest code.
6525	*
6526	* @returns VBox status code.
6527	* @param pVCpu The cross context virtual CPU structure.
6528	* @param pVmxTransient The VMX-transient structure.
6529	*
6530	* @remarks No-long-jump zone!!!
6531	*/
6532	static int hmR0VmxSelectVMRunHandler(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6533	{
6534	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6535	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6536
6537	if (CPUMIsGuestInLongModeEx(pCtx))
6538	{
6539	#ifndef VBOX_WITH_64_BITS_GUESTS
6540	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6541	#else
6542	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6543	/* Guest is in long mode, use the 64-bit handler (host is 64-bit). */
6544	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6545	#endif
6546	}
6547	else
6548	{
6549	/* Guest is not in long mode, use the 32-bit handler. */
6550	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6551	}
6552	Assert(pVmcsInfo->pfnStartVM);
6553	return VINF_SUCCESS;
6554	}
6555
6556
6557	/**
6558	* Wrapper for running the guest code in VT-x.
6559	*
6560	* @returns VBox status code, no informational status codes.
6561	* @param pVCpu The cross context virtual CPU structure.
6562	* @param pVmxTransient The VMX-transient structure.
6563	*
6564	* @remarks No-long-jump zone!!!
6565	*/
6566	DECLINLINE(int) hmR0VmxRunGuest(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6567	{
6568	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6569	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6570	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6571
6572	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6573
6574	/*
6575	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6576	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6577	* callee-saved and thus the need for this XMM wrapper.
6578	*
6579	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6580	*/
6581	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6582	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6583	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6584	#ifdef VBOX_WITH_KERNEL_USING_XMM
6585	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6586	#else
6587	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu);
6588	#endif
6589	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6590	return rc;
6591	}
6592
6593
6594	/**
6595	* Reports world-switch error and dumps some useful debug info.
6596	*
6597	* @param pVCpu The cross context virtual CPU structure.
6598	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6599	* @param pVmxTransient The VMX-transient structure (only
6600	* exitReason updated).
6601	*/
6602	static void hmR0VmxReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6603	{
6604	Assert(pVCpu);
6605	Assert(pVmxTransient);
6606	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6607
6608	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6609	switch (rcVMRun)
6610	{
6611	case VERR_VMX_INVALID_VMXON_PTR:
6612	AssertFailed();
6613	break;
6614	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6615	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6616	{
6617	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6618	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6619	AssertRC(rc);
6620	hmR0VmxReadExitQualVmcs(pVmxTransient);
6621
6622	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6623	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6624	Cannot do it here as we may have been long preempted. */
6625
6626	#ifdef VBOX_STRICT
6627	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6628	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6629	pVmxTransient->uExitReason));
6630	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6631	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6632	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6633	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6634	else
6635	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6636	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6637	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6638
6639	static struct
6640	{
6641	/** Name of the field to log. */
6642	const char *pszName;
6643	/** The VMCS field. */
6644	uint32_t uVmcsField;
6645	/** Whether host support of this field needs to be checked. */
6646	bool fCheckSupport;
6647	} const s_aVmcsFields[] =
6648	{
6649	{ "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false },
6650	{ "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false },
6651	{ "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true },
6652	{ "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false },
6653	{ "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false },
6654	{ "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false },
6655	{ "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false },
6656	{ "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false },
6657	{ "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false },
6658	{ "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false },
6659	{ "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false },
6660	{ "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false },
6661	{ "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false },
6662	{ "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false },
6663	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false },
6664	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false },
6665	{ "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false },
6666	{ "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false },
6667	{ "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false },
6668	{ "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false },
6669	{ "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true },
6670	{ "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false },
6671	{ "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false },
6672	{ "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false },
6673	{ "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
6674	{ "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false },
6675	{ "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false },
6676	{ "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false },
6677	/* The order of selector fields below are fixed! */
6678	{ "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false },
6679	{ "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false },
6680	{ "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false },
6681	{ "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false },
6682	{ "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false },
6683	{ "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false },
6684	{ "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false },
6685	/* End of ordered selector fields. */
6686	{ "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false },
6687	{ "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false },
6688	{ "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false },
6689	{ "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false },
6690	{ "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false },
6691	{ "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false },
6692	{ "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false },
6693	{ "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false }
6694	};
6695
6696	RTGDTR HostGdtr;
6697	ASMGetGDTR(&HostGdtr);
6698
6699	uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
6700	for (uint32_t i = 0; i < cVmcsFields; i++)
6701	{
6702	uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
6703
6704	bool fSupported;
6705	if (!s_aVmcsFields[i].fCheckSupport)
6706	fSupported = true;
6707	else
6708	{
6709	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6710	switch (uVmcsField)
6711	{
6712	case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hm.s.fNestedPaging; break;
6713	case VMX_VMCS16_VPID: fSupported = pVM->hm.s.vmx.fVpid; break;
6714	case VMX_VMCS32_CTRL_PROC_EXEC2:
6715	fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6716	break;
6717	default:
6718	AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
6719	}
6720	}
6721
6722	if (fSupported)
6723	{
6724	uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
6725	switch (uWidth)
6726	{
6727	case VMX_VMCSFIELD_WIDTH_16BIT:
6728	{
6729	uint16_t u16Val;
6730	rc = VMXReadVmcs16(uVmcsField, &u16Val);
6731	AssertRC(rc);
6732	Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
6733
6734	if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
6735	&& uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
6736	{
6737	if (u16Val < HostGdtr.cbGdt)
6738	{
6739	/* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
6740	static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
6741	"Host FS", "Host GS", "Host TR" };
6742	uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
6743	Assert(idxSel < RT_ELEMENTS(s_apszSel));
6744	PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
6745	hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
6746	}
6747	else
6748	Log4((" Selector value exceeds GDT limit!\n"));
6749	}
6750	break;
6751	}
6752
6753	case VMX_VMCSFIELD_WIDTH_32BIT:
6754	{
6755	uint32_t u32Val;
6756	rc = VMXReadVmcs32(uVmcsField, &u32Val);
6757	AssertRC(rc);
6758	Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
6759	break;
6760	}
6761
6762	case VMX_VMCSFIELD_WIDTH_64BIT:
6763	case VMX_VMCSFIELD_WIDTH_NATURAL:
6764	{
6765	uint64_t u64Val;
6766	rc = VMXReadVmcs64(uVmcsField, &u64Val);
6767	AssertRC(rc);
6768	Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
6769	break;
6770	}
6771	}
6772	}
6773	}
6774
6775	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6776	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6777	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6778	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6779	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6780	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6781	#endif /* VBOX_STRICT */
6782	break;
6783	}
6784
6785	default:
6786	/* Impossible */
6787	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6788	break;
6789	}
6790	}
6791
6792
6793	/**
6794	* Sets up the usage of TSC-offsetting and updates the VMCS.
6795	*
6796	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6797	* VMX-preemption timer.
6798	*
6799	* @returns VBox status code.
6800	* @param pVCpu The cross context virtual CPU structure.
6801	* @param pVmxTransient The VMX-transient structure.
6802	*
6803	* @remarks No-long-jump zone!!!
6804	*/
6805	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6806	{
6807	bool fOffsettedTsc;
6808	bool fParavirtTsc;
6809	uint64_t uTscOffset;
6810	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6811	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6812
6813	if (pVM->hm.s.vmx.fUsePreemptTimer)
6814	{
6815	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6816
6817	/* Make sure the returned values have sane upper and lower boundaries. */
6818	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6819	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6820	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6821	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6822
6823	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6824	* preemption timers here. We probably need to clamp the preemption timer,
6825	* after converting the timer value to the host. */
6826	uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6827	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6828	AssertRC(rc);
6829	}
6830	else
6831	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6832
6833	if (fParavirtTsc)
6834	{
6835	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6836	information before every VM-entry, hence disable it for performance sake. */
6837	#if 0
6838	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6839	AssertRC(rc);
6840	#endif
6841	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6842	}
6843
6844	if ( fOffsettedTsc
6845	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6846	{
6847	if (pVmxTransient->fIsNestedGuest)
6848	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6849	hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
6850	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6851	}
6852	else
6853	{
6854	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6855	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6856	}
6857	}
6858
6859
6860	/**
6861	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6862	* VM-exit interruption info type.
6863	*
6864	* @returns The IEM exception flags.
6865	* @param uVector The event vector.
6866	* @param uVmxEventType The VMX event type.
6867	*
6868	* @remarks This function currently only constructs flags required for
6869	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6870	* and CR2 aspects of an exception are not included).
6871	*/
6872	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6873	{
6874	uint32_t fIemXcptFlags;
6875	switch (uVmxEventType)
6876	{
6877	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6878	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6879	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6880	break;
6881
6882	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6883	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6884	break;
6885
6886	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6887	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6888	break;
6889
6890	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6891	{
6892	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6893	if (uVector == X86_XCPT_BP)
6894	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6895	else if (uVector == X86_XCPT_OF)
6896	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6897	else
6898	{
6899	fIemXcptFlags = 0;
6900	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6901	}
6902	break;
6903	}
6904
6905	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6906	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6907	break;
6908
6909	default:
6910	fIemXcptFlags = 0;
6911	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6912	break;
6913	}
6914	return fIemXcptFlags;
6915	}
6916
6917
6918	/**
6919	* Sets an event as a pending event to be injected into the guest.
6920	*
6921	* @param pVCpu The cross context virtual CPU structure.
6922	* @param u32IntInfo The VM-entry interruption-information field.
6923	* @param cbInstr The VM-entry instruction length in bytes (for
6924	* software interrupts, exceptions and privileged
6925	* software exceptions).
6926	* @param u32ErrCode The VM-entry exception error code.
6927	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6928	* page-fault.
6929	*/
6930	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6931	RTGCUINTPTR GCPtrFaultAddress)
6932	{
6933	Assert(!pVCpu->hm.s.Event.fPending);
6934	pVCpu->hm.s.Event.fPending = true;
6935	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6936	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6937	pVCpu->hm.s.Event.cbInstr = cbInstr;
6938	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6939	}
6940
6941
6942	/**
6943	* Sets an external interrupt as pending-for-injection into the VM.
6944	*
6945	* @param pVCpu The cross context virtual CPU structure.
6946	* @param u8Interrupt The external interrupt vector.
6947	*/
6948	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
6949	{
6950	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6951	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6952	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6953	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6954	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6955	}
6956
6957
6958	/**
6959	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6960	*
6961	* @param pVCpu The cross context virtual CPU structure.
6962	*/
6963	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPUCC pVCpu)
6964	{
6965	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6966	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6967	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6968	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6969	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6970	}
6971
6972
6973	/**
6974	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6975	*
6976	* @param pVCpu The cross context virtual CPU structure.
6977	*/
6978	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPUCC pVCpu)
6979	{
6980	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6981	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6982	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6983	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6984	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6985	}
6986
6987
6988	/**
6989	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6990	*
6991	* @param pVCpu The cross context virtual CPU structure.
6992	*/
6993	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPUCC pVCpu)
6994	{
6995	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6996	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6997	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6998	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6999	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7000	}
7001
7002
7003	/**
7004	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7005	*
7006	* @param pVCpu The cross context virtual CPU structure.
7007	*/
7008	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPUCC pVCpu)
7009	{
7010	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7011	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7012	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7013	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7014	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7015	}
7016
7017
7018	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7019	/**
7020	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7021	*
7022	* @param pVCpu The cross context virtual CPU structure.
7023	* @param u32ErrCode The error code for the general-protection exception.
7024	*/
7025	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7026	{
7027	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7028	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7029	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7030	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7031	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7032	}
7033
7034
7035	/**
7036	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7037	*
7038	* @param pVCpu The cross context virtual CPU structure.
7039	* @param u32ErrCode The error code for the stack exception.
7040	*/
7041	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7042	{
7043	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7044	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7045	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7046	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7047	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7048	}
7049	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7050
7051
7052	/**
7053	* Fixes up attributes for the specified segment register.
7054	*
7055	* @param pVCpu The cross context virtual CPU structure.
7056	* @param pSelReg The segment register that needs fixing.
7057	* @param idxSel The VMCS field for the corresponding segment register.
7058	*/
7059	static void hmR0VmxFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7060	{
7061	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7062
7063	/*
7064	* If VT-x marks the segment as unusable, most other bits remain undefined:
7065	* - For CS the L, D and G bits have meaning.
7066	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7067	* - For the remaining data segments no bits are defined.
7068	*
7069	* The present bit and the unusable bit has been observed to be set at the
7070	* same time (the selector was supposed to be invalid as we started executing
7071	* a V8086 interrupt in ring-0).
7072	*
7073	* What should be important for the rest of the VBox code, is that the P bit is
7074	* cleared. Some of the other VBox code recognizes the unusable bit, but
7075	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7076	* safe side here, we'll strip off P and other bits we don't care about. If
7077	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7078	*
7079	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7080	*/
7081	#ifdef VBOX_STRICT
7082	uint32_t const uAttr = pSelReg->Attr.u;
7083	#endif
7084
7085	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7086	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7087	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7088
7089	#ifdef VBOX_STRICT
7090	VMMRZCallRing3Disable(pVCpu);
7091	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7092	# ifdef DEBUG_bird
7093	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7094	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7095	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7096	# endif
7097	VMMRZCallRing3Enable(pVCpu);
7098	NOREF(uAttr);
7099	#endif
7100	RT_NOREF2(pVCpu, idxSel);
7101	}
7102
7103
7104	/**
7105	* Imports a guest segment register from the current VMCS into the guest-CPU
7106	* context.
7107	*
7108	* @param pVCpu The cross context virtual CPU structure.
7109	* @param iSegReg The segment register number (X86_SREG_XXX).
7110	*
7111	* @remarks Called with interrupts and/or preemption disabled.
7112	*/
7113	static void hmR0VmxImportGuestSegReg(PVMCPUCC pVCpu, uint8_t iSegReg)
7114	{
7115	Assert(iSegReg < X86_SREG_COUNT);
7116
7117	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7118	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7119	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7120	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7121
7122	uint16_t u16Sel;
7123	uint64_t u64Base;
7124	uint32_t u32Limit, u32Attr;
7125	int rc = VMXReadVmcs16(idxSel, &u16Sel); AssertRC(rc);
7126	rc = VMXReadVmcs32(idxLimit, &u32Limit); AssertRC(rc);
7127	rc = VMXReadVmcs32(idxAttr, &u32Attr); AssertRC(rc);
7128	rc = VMXReadVmcsNw(idxBase, &u64Base); AssertRC(rc);
7129
7130	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7131	pSelReg->Sel = u16Sel;
7132	pSelReg->ValidSel = u16Sel;
7133	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7134	pSelReg->u32Limit = u32Limit;
7135	pSelReg->u64Base = u64Base;
7136	pSelReg->Attr.u = u32Attr;
7137	if (u32Attr & X86DESCATTR_UNUSABLE)
7138	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7139	}
7140
7141
7142	/**
7143	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7144	*
7145	* @param pVCpu The cross context virtual CPU structure.
7146	*
7147	* @remarks Called with interrupts and/or preemption disabled.
7148	*/
7149	static void hmR0VmxImportGuestLdtr(PVMCPUCC pVCpu)
7150	{
7151	uint16_t u16Sel;
7152	uint64_t u64Base;
7153	uint32_t u32Limit, u32Attr;
7154	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
7155	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
7156	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7157	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
7158
7159	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
7160	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
7161	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7162	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7163	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7164	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7165	if (u32Attr & X86DESCATTR_UNUSABLE)
7166	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7167	}
7168
7169
7170	/**
7171	* Imports the guest TR from the current VMCS into the guest-CPU context.
7172	*
7173	* @param pVCpu The cross context virtual CPU structure.
7174	*
7175	* @remarks Called with interrupts and/or preemption disabled.
7176	*/
7177	static void hmR0VmxImportGuestTr(PVMCPUCC pVCpu)
7178	{
7179	uint16_t u16Sel;
7180	uint64_t u64Base;
7181	uint32_t u32Limit, u32Attr;
7182	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
7183	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
7184	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7185	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
7186
7187	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
7188	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
7189	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7190	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7191	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7192	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7193	/* TR is the only selector that can never be unusable. */
7194	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7195	}
7196
7197
7198	/**
7199	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7200	*
7201	* @param pVCpu The cross context virtual CPU structure.
7202	*
7203	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7204	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7205	* instead!!!
7206	*/
7207	static void hmR0VmxImportGuestRip(PVMCPUCC pVCpu)
7208	{
7209	uint64_t u64Val;
7210	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7211	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7212	{
7213	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
7214	AssertRC(rc);
7215
7216	pCtx->rip = u64Val;
7217	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7218	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7219	}
7220	}
7221
7222
7223	/**
7224	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7225	*
7226	* @param pVCpu The cross context virtual CPU structure.
7227	* @param pVmcsInfo The VMCS info. object.
7228	*
7229	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7230	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7231	* instead!!!
7232	*/
7233	static void hmR0VmxImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7234	{
7235	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7236	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7237	{
7238	uint64_t u64Val;
7239	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7240	AssertRC(rc);
7241
7242	pCtx->rflags.u64 = u64Val;
7243	if (pVmcsInfo->RealMode.fRealOnV86Active)
7244	{
7245	pCtx->eflags.Bits.u1VM = 0;
7246	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7247	}
7248	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7249	}
7250	}
7251
7252
7253	/**
7254	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7255	* context.
7256	*
7257	* @param pVCpu The cross context virtual CPU structure.
7258	* @param pVmcsInfo The VMCS info. object.
7259	*
7260	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7261	* do not log!
7262	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7263	* instead!!!
7264	*/
7265	static void hmR0VmxImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7266	{
7267	uint32_t u32Val;
7268	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
7269	if (!u32Val)
7270	{
7271	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7272	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7273	CPUMSetGuestNmiBlocking(pVCpu, false);
7274	}
7275	else
7276	{
7277	/*
7278	* We must import RIP here to set our EM interrupt-inhibited state.
7279	* We also import RFLAGS as our code that evaluates pending interrupts
7280	* before VM-entry requires it.
7281	*/
7282	hmR0VmxImportGuestRip(pVCpu);
7283	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7284
7285	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7286	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7287	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7288	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7289
7290	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7291	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7292	}
7293	}
7294
7295
7296	/**
7297	* Worker for VMXR0ImportStateOnDemand.
7298	*
7299	* @returns VBox status code.
7300	* @param pVCpu The cross context virtual CPU structure.
7301	* @param pVmcsInfo The VMCS info. object.
7302	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7303	*/
7304	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7305	{
7306	int rc = VINF_SUCCESS;
7307	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7308	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7309	uint32_t u32Val;
7310
7311	/*
7312	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7313	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7314	* neither are other host platforms.
7315	*
7316	* Committing this temporarily as it prevents BSOD.
7317	*
7318	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7319	*/
7320	#ifdef RT_OS_WINDOWS
7321	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7322	return VERR_HM_IPE_1;
7323	#endif
7324
7325	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7326
7327	/*
7328	* We disable interrupts to make the updating of the state and in particular
7329	* the fExtrn modification atomic wrt to preemption hooks.
7330	*/
7331	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7332
7333	fWhat &= pCtx->fExtrn;
7334	if (fWhat)
7335	{
7336	do
7337	{
7338	if (fWhat & CPUMCTX_EXTRN_RIP)
7339	hmR0VmxImportGuestRip(pVCpu);
7340
7341	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7342	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7343
7344	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7345	hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7346
7347	if (fWhat & CPUMCTX_EXTRN_RSP)
7348	{
7349	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &pCtx->rsp);
7350	AssertRC(rc);
7351	}
7352
7353	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7354	{
7355	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7356	if (fWhat & CPUMCTX_EXTRN_CS)
7357	{
7358	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7359	hmR0VmxImportGuestRip(pVCpu);
7360	if (fRealOnV86Active)
7361	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7362	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7363	}
7364	if (fWhat & CPUMCTX_EXTRN_SS)
7365	{
7366	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7367	if (fRealOnV86Active)
7368	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7369	}
7370	if (fWhat & CPUMCTX_EXTRN_DS)
7371	{
7372	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7373	if (fRealOnV86Active)
7374	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7375	}
7376	if (fWhat & CPUMCTX_EXTRN_ES)
7377	{
7378	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7379	if (fRealOnV86Active)
7380	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7381	}
7382	if (fWhat & CPUMCTX_EXTRN_FS)
7383	{
7384	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7385	if (fRealOnV86Active)
7386	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7387	}
7388	if (fWhat & CPUMCTX_EXTRN_GS)
7389	{
7390	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7391	if (fRealOnV86Active)
7392	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7393	}
7394	}
7395
7396	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7397	{
7398	if (fWhat & CPUMCTX_EXTRN_LDTR)
7399	hmR0VmxImportGuestLdtr(pVCpu);
7400
7401	if (fWhat & CPUMCTX_EXTRN_GDTR)
7402	{
7403	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
7404	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
7405	pCtx->gdtr.cbGdt = u32Val;
7406	}
7407
7408	/* Guest IDTR. */
7409	if (fWhat & CPUMCTX_EXTRN_IDTR)
7410	{
7411	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
7412	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
7413	pCtx->idtr.cbIdt = u32Val;
7414	}
7415
7416	/* Guest TR. */
7417	if (fWhat & CPUMCTX_EXTRN_TR)
7418	{
7419	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7420	don't need to import that one. */
7421	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7422	hmR0VmxImportGuestTr(pVCpu);
7423	}
7424	}
7425
7426	if (fWhat & CPUMCTX_EXTRN_DR7)
7427	{
7428	if (!pVCpu->hm.s.fUsingHyperDR7)
7429	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &pCtx->dr[7]); AssertRC(rc);
7430	}
7431
7432	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7433	{
7434	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
7435	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
7436	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
7437	pCtx->SysEnter.cs = u32Val;
7438	}
7439
7440	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7441	{
7442	if ( pVM->hm.s.fAllow64BitGuests
7443	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7444	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7445	}
7446
7447	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7448	{
7449	if ( pVM->hm.s.fAllow64BitGuests
7450	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7451	{
7452	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7453	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7454	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7455	}
7456	}
7457
7458	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7459	{
7460	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7461	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7462	Assert(pMsrs);
7463	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7464	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7465	for (uint32_t i = 0; i < cMsrs; i++)
7466	{
7467	uint32_t const idMsr = pMsrs[i].u32Msr;
7468	switch (idMsr)
7469	{
7470	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7471	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7472	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7473	default:
7474	{
7475	pCtx->fExtrn = 0;
7476	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7477	ASMSetFlags(fEFlags);
7478	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7479	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7480	}
7481	}
7482	}
7483	}
7484
7485	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7486	{
7487	if (fWhat & CPUMCTX_EXTRN_CR0)
7488	{
7489	uint64_t u64Cr0;
7490	uint64_t u64Shadow;
7491	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
7492	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
7493	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7494	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7495	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7496	#else
7497	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7498	{
7499	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7500	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7501	}
7502	else
7503	{
7504	/*
7505	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
7506	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7507	* re-construct CR0. See @bugref{9180#c95} for details.
7508	*/
7509	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7510	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7511	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7512	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
7513	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
7514	}
7515	#endif
7516	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7517	CPUMSetGuestCR0(pVCpu, u64Cr0);
7518	VMMRZCallRing3Enable(pVCpu);
7519	}
7520
7521	if (fWhat & CPUMCTX_EXTRN_CR4)
7522	{
7523	uint64_t u64Cr4;
7524	uint64_t u64Shadow;
7525	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
7526	rc \|= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
7527	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7528	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7529	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7530	#else
7531	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7532	{
7533	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7534	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7535	}
7536	else
7537	{
7538	/*
7539	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
7540	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7541	* re-construct CR4. See @bugref{9180#c95} for details.
7542	*/
7543	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7544	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7545	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7546	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
7547	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
7548	}
7549	#endif
7550	pCtx->cr4 = u64Cr4;
7551	}
7552
7553	if (fWhat & CPUMCTX_EXTRN_CR3)
7554	{
7555	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7556	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7557	\|\| ( pVM->hm.s.fNestedPaging
7558	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7559	{
7560	uint64_t u64Cr3;
7561	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
7562	if (pCtx->cr3 != u64Cr3)
7563	{
7564	pCtx->cr3 = u64Cr3;
7565	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7566	}
7567
7568	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7569	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7570	if (CPUMIsGuestInPAEModeEx(pCtx))
7571	{
7572	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
7573	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
7574	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
7575	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
7576	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7577	}
7578	}
7579	}
7580	}
7581
7582	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7583	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7584	{
7585	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7586	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7587	{
7588	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7589	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7590	if (RT_SUCCESS(rc))
7591	{ /* likely */ }
7592	else
7593	break;
7594	}
7595	}
7596	#endif
7597	} while (0);
7598
7599	if (RT_SUCCESS(rc))
7600	{
7601	/* Update fExtrn. */
7602	pCtx->fExtrn &= ~fWhat;
7603
7604	/* If everything has been imported, clear the HM keeper bit. */
7605	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7606	{
7607	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7608	Assert(!pCtx->fExtrn);
7609	}
7610	}
7611	}
7612	else
7613	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7614
7615	/*
7616	* Restore interrupts.
7617	*/
7618	ASMSetFlags(fEFlags);
7619
7620	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7621
7622	if (RT_SUCCESS(rc))
7623	{ /* likely */ }
7624	else
7625	return rc;
7626
7627	/*
7628	* Honor any pending CR3 updates.
7629	*
7630	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
7631	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7632	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7633	*
7634	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7635	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7636	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7637	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7638	*
7639	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7640	*/
7641	if (VMMRZCallRing3IsEnabled(pVCpu))
7642	{
7643	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7644	{
7645	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7646	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7647	}
7648
7649	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7650	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7651
7652	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7653	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7654	}
7655
7656	return VINF_SUCCESS;
7657	}
7658
7659
7660	/**
7661	* Saves the guest state from the VMCS into the guest-CPU context.
7662	*
7663	* @returns VBox status code.
7664	* @param pVCpu The cross context virtual CPU structure.
7665	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7666	*/
7667	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
7668	{
7669	AssertPtr(pVCpu);
7670	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7671	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7672	}
7673
7674
7675	/**
7676	* Check per-VM and per-VCPU force flag actions that require us to go back to
7677	* ring-3 for one reason or another.
7678	*
7679	* @returns Strict VBox status code (i.e. informational status codes too)
7680	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7681	* ring-3.
7682	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7683	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7684	* interrupts)
7685	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7686	* all EMTs to be in ring-3.
7687	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7688	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7689	* to the EM loop.
7690	*
7691	* @param pVCpu The cross context virtual CPU structure.
7692	* @param fStepping Whether we are single-stepping the guest using the
7693	* hypervisor debugger.
7694	*
7695	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7696	* is no longer in VMX non-root mode.
7697	*/
7698	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPUCC pVCpu, bool fStepping)
7699	{
7700	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7701
7702	/*
7703	* Update pending interrupts into the APIC's IRR.
7704	*/
7705	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7706	APICUpdatePendingInterrupts(pVCpu);
7707
7708	/*
7709	* Anything pending? Should be more likely than not if we're doing a good job.
7710	*/
7711	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7712	if ( !fStepping
7713	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7714	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7715	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7716	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7717	return VINF_SUCCESS;
7718
7719	/* Pending PGM C3 sync. */
7720	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7721	{
7722	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7723	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7724	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7725	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7726	if (rcStrict != VINF_SUCCESS)
7727	{
7728	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7729	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7730	return rcStrict;
7731	}
7732	}
7733
7734	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7735	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7736	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7737	{
7738	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7739	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7740	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7741	return rc;
7742	}
7743
7744	/* Pending VM request packets, such as hardware interrupts. */
7745	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7746	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7747	{
7748	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7749	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7750	return VINF_EM_PENDING_REQUEST;
7751	}
7752
7753	/* Pending PGM pool flushes. */
7754	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7755	{
7756	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7757	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7758	return VINF_PGM_POOL_FLUSH_PENDING;
7759	}
7760
7761	/* Pending DMA requests. */
7762	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7763	{
7764	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7765	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7766	return VINF_EM_RAW_TO_R3;
7767	}
7768
7769	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7770	/* Pending nested-guest APIC-write (has highest priority among nested-guest FFs). */
7771	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7772	{
7773	Log4Func(("Pending nested-guest APIC-write\n"));
7774	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
7775	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7776	return rcStrict;
7777	}
7778	/** @todo VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_PREEMPT_TIMER */
7779	#endif
7780
7781	return VINF_SUCCESS;
7782	}
7783
7784
7785	/**
7786	* Converts any TRPM trap into a pending HM event. This is typically used when
7787	* entering from ring-3 (not longjmp returns).
7788	*
7789	* @param pVCpu The cross context virtual CPU structure.
7790	*/
7791	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
7792	{
7793	Assert(TRPMHasTrap(pVCpu));
7794	Assert(!pVCpu->hm.s.Event.fPending);
7795
7796	uint8_t uVector;
7797	TRPMEVENT enmTrpmEvent;
7798	uint32_t uErrCode;
7799	RTGCUINTPTR GCPtrFaultAddress;
7800	uint8_t cbInstr;
7801	bool fIcebp;
7802
7803	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
7804	AssertRC(rc);
7805
7806	uint32_t u32IntInfo;
7807	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
7808	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
7809
7810	rc = TRPMResetTrap(pVCpu);
7811	AssertRC(rc);
7812	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7813	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7814
7815	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7816	}
7817
7818
7819	/**
7820	* Converts the pending HM event into a TRPM trap.
7821	*
7822	* @param pVCpu The cross context virtual CPU structure.
7823	*/
7824	static void hmR0VmxPendingEventToTrpmTrap(PVMCPUCC pVCpu)
7825	{
7826	Assert(pVCpu->hm.s.Event.fPending);
7827
7828	/* If a trap was already pending, we did something wrong! */
7829	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7830
7831	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
7832	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
7833	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
7834
7835	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7836
7837	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7838	AssertRC(rc);
7839
7840	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7841	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
7842
7843	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
7844	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7845	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
7846	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7847
7848	if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7849	TRPMSetTrapDueToIcebp(pVCpu);
7850
7851	/* We're now done converting the pending event. */
7852	pVCpu->hm.s.Event.fPending = false;
7853	}
7854
7855
7856	/**
7857	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7858	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7859	*
7860	* @param pVCpu The cross context virtual CPU structure.
7861	* @param pVmcsInfo The VMCS info. object.
7862	*/
7863	static void hmR0VmxSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7864	{
7865	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7866	{
7867	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7868	{
7869	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7870	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7871	AssertRC(rc);
7872	}
7873	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7874	}
7875
7876
7877	/**
7878	* Clears the interrupt-window exiting control in the VMCS.
7879	*
7880	* @param pVmcsInfo The VMCS info. object.
7881	*/
7882	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7883	{
7884	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7885	{
7886	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7887	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7888	AssertRC(rc);
7889	}
7890	}
7891
7892
7893	/**
7894	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7895	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7896	*
7897	* @param pVCpu The cross context virtual CPU structure.
7898	* @param pVmcsInfo The VMCS info. object.
7899	*/
7900	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7901	{
7902	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7903	{
7904	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7905	{
7906	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7907	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7908	AssertRC(rc);
7909	Log4Func(("Setup NMI-window exiting\n"));
7910	}
7911	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7912	}
7913
7914
7915	/**
7916	* Clears the NMI-window exiting control in the VMCS.
7917	*
7918	* @param pVmcsInfo The VMCS info. object.
7919	*/
7920	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7921	{
7922	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7923	{
7924	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7925	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7926	AssertRC(rc);
7927	}
7928	}
7929
7930
7931	/**
7932	* Does the necessary state syncing before returning to ring-3 for any reason
7933	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7934	*
7935	* @returns VBox status code.
7936	* @param pVCpu The cross context virtual CPU structure.
7937	* @param fImportState Whether to import the guest state from the VMCS back
7938	* to the guest-CPU context.
7939	*
7940	* @remarks No-long-jmp zone!!!
7941	*/
7942	static int hmR0VmxLeave(PVMCPUCC pVCpu, bool fImportState)
7943	{
7944	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7945	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7946
7947	RTCPUID const idCpu = RTMpCpuId();
7948	Log4Func(("HostCpuId=%u\n", idCpu));
7949
7950	/*
7951	* !!! IMPORTANT !!!
7952	* If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
7953	*/
7954
7955	/* Save the guest state if necessary. */
7956	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7957	if (fImportState)
7958	{
7959	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
7960	AssertRCReturn(rc, rc);
7961	}
7962
7963	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7964	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7965	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7966
7967	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7968	#ifdef VBOX_STRICT
7969	if (CPUMIsHyperDebugStateActive(pVCpu))
7970	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7971	#endif
7972	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7973	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7974	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7975
7976	/* Restore host-state bits that VT-x only restores partially. */
7977	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7978	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7979	{
7980	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7981	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7982	}
7983	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7984
7985	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7986	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7987	{
7988	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7989	if (!fImportState)
7990	{
7991	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
7992	AssertRCReturn(rc, rc);
7993	}
7994	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7995	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7996	}
7997	else
7998	pVCpu->hm.s.vmx.fLazyMsrs = 0;
7999
8000	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8001	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8002
8003	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8004	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8005	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8006	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8007	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8008	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8009	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8010	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8011	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8012	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8013
8014	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8015
8016	/** @todo This partially defeats the purpose of having preemption hooks.
8017	* The problem is, deregistering the hooks should be moved to a place that
8018	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8019	* context.
8020	*/
8021	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8022	AssertRCReturn(rc, rc);
8023
8024	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8025	/*
8026	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8027	* clear a shadow VMCS before allowing that VMCS to become active on another
8028	* logical processor. We may or may not be importing guest state which clears
8029	* it, so cover for it here.
8030	*
8031	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8032	*/
8033	if ( pVmcsInfo->pvShadowVmcs
8034	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8035	{
8036	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8037	AssertRCReturn(rc, rc);
8038	}
8039
8040	/*
8041	* Flag that we need to re-export the host state if we switch to this VMCS before
8042	* executing guest or nested-guest code.
8043	*/
8044	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
8045	#endif
8046
8047	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8048	NOREF(idCpu);
8049	return VINF_SUCCESS;
8050	}
8051
8052
8053	/**
8054	* Leaves the VT-x session.
8055	*
8056	* @returns VBox status code.
8057	* @param pVCpu The cross context virtual CPU structure.
8058	*
8059	* @remarks No-long-jmp zone!!!
8060	*/
8061	static int hmR0VmxLeaveSession(PVMCPUCC pVCpu)
8062	{
8063	HM_DISABLE_PREEMPT(pVCpu);
8064	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8065	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8066	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8067
8068	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8069	and done this from the VMXR0ThreadCtxCallback(). */
8070	if (!pVCpu->hm.s.fLeaveDone)
8071	{
8072	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8073	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8074	pVCpu->hm.s.fLeaveDone = true;
8075	}
8076	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8077
8078	/*
8079	* !!! IMPORTANT !!!
8080	* If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
8081	*/
8082
8083	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8084	/** @todo Deregistering here means we need to VMCLEAR always
8085	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8086	* for calling VMMR0ThreadCtxHookDisable here! */
8087	VMMR0ThreadCtxHookDisable(pVCpu);
8088
8089	/* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
8090	int rc = HMR0LeaveCpu(pVCpu);
8091	HM_RESTORE_PREEMPT();
8092	return rc;
8093	}
8094
8095
8096	/**
8097	* Does the necessary state syncing before doing a longjmp to ring-3.
8098	*
8099	* @returns VBox status code.
8100	* @param pVCpu The cross context virtual CPU structure.
8101	*
8102	* @remarks No-long-jmp zone!!!
8103	*/
8104	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPUCC pVCpu)
8105	{
8106	return hmR0VmxLeaveSession(pVCpu);
8107	}
8108
8109
8110	/**
8111	* Take necessary actions before going back to ring-3.
8112	*
8113	* An action requires us to go back to ring-3. This function does the necessary
8114	* steps before we can safely return to ring-3. This is not the same as longjmps
8115	* to ring-3, this is voluntary and prepares the guest so it may continue
8116	* executing outside HM (recompiler/IEM).
8117	*
8118	* @returns VBox status code.
8119	* @param pVCpu The cross context virtual CPU structure.
8120	* @param rcExit The reason for exiting to ring-3. Can be
8121	* VINF_VMM_UNKNOWN_RING3_CALL.
8122	*/
8123	static int hmR0VmxExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
8124	{
8125	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8126
8127	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8128	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8129	{
8130	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8131	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8132	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8133	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8134	}
8135
8136	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8137	VMMRZCallRing3Disable(pVCpu);
8138	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8139
8140	/*
8141	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8142	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8143	*
8144	* This is because execution may continue from ring-3 and we would need to inject
8145	* the event from there (hence place it back in TRPM).
8146	*/
8147	if (pVCpu->hm.s.Event.fPending)
8148	{
8149	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8150	Assert(!pVCpu->hm.s.Event.fPending);
8151
8152	/* Clear the events from the VMCS. */
8153	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8154	AssertRC(rc);
8155	}
8156	#ifdef VBOX_STRICT
8157	else
8158	{
8159	/*
8160	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8161	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8162	* occasionally, see @bugref{9180#c42}.
8163	*
8164	* However, if the VM-entry failed, any VM entry-interruption info. field would
8165	* be left unmodified as the event would not have been injected to the guest. In
8166	* such cases, don't assert, we're not going to continue guest execution anyway.
8167	*/
8168	uint32_t uExitReason;
8169	uint32_t uEntryIntInfo;
8170	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8171	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8172	AssertRC(rc);
8173	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8174	}
8175	#endif
8176
8177	/*
8178	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8179	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8180	* (e.g. TPR below threshold).
8181	*/
8182	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8183	{
8184	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8185	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8186	}
8187
8188	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8189	and if we're injecting an event we should have a TRPM trap pending. */
8190	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8191	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8192	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8193	#endif
8194
8195	/* Save guest state and restore host state bits. */
8196	int rc = hmR0VmxLeaveSession(pVCpu);
8197	AssertRCReturn(rc, rc);
8198	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8199
8200	/* Thread-context hooks are unregistered at this point!!! */
8201	/* Ring-3 callback notifications are unregistered at this point!!! */
8202
8203	/* Sync recompiler state. */
8204	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8205	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8206	\| CPUM_CHANGED_LDTR
8207	\| CPUM_CHANGED_GDTR
8208	\| CPUM_CHANGED_IDTR
8209	\| CPUM_CHANGED_TR
8210	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8211	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8212	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8213	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8214
8215	Assert(!pVCpu->hm.s.fClearTrapFlag);
8216
8217	/* Update the exit-to-ring 3 reason. */
8218	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8219
8220	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8221	if ( rcExit != VINF_EM_RAW_INTERRUPT
8222	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8223	{
8224	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8225	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8226	}
8227
8228	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8229	VMMRZCallRing3Enable(pVCpu);
8230	return rc;
8231	}
8232
8233
8234	/**
8235	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8236	* longjump to ring-3 and possibly get preempted.
8237	*
8238	* @returns VBox status code.
8239	* @param pVCpu The cross context virtual CPU structure.
8240	* @param enmOperation The operation causing the ring-3 longjump.
8241	*/
8242	VMMR0DECL(int) VMXR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
8243	{
8244	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8245	{
8246	/*
8247	* !!! IMPORTANT !!!
8248	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8249	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8250	*/
8251	VMMRZCallRing3RemoveNotification(pVCpu);
8252	VMMRZCallRing3Disable(pVCpu);
8253	HM_DISABLE_PREEMPT(pVCpu);
8254
8255	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8256	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8257	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8258	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8259
8260	/* Restore host-state bits that VT-x only restores partially. */
8261	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8262	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8263	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8264	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8265
8266	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8267	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8268	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8269
8270	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8271	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8272	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8273
8274	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8275	cleared as part of importing the guest state above. */
8276	hmR0VmxClearVmcs(pVmcsInfo);
8277
8278	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8279	VMMR0ThreadCtxHookDisable(pVCpu);
8280
8281	/* Leave HM context. This takes care of local init (term). */
8282	HMR0LeaveCpu(pVCpu);
8283	HM_RESTORE_PREEMPT();
8284	return VINF_SUCCESS;
8285	}
8286
8287	Assert(pVCpu);
8288	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8289	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8290
8291	VMMRZCallRing3Disable(pVCpu);
8292	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8293
8294	Log4Func(("-> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8295
8296	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8297	AssertRCReturn(rc, rc);
8298
8299	VMMRZCallRing3Enable(pVCpu);
8300	return VINF_SUCCESS;
8301	}
8302
8303
8304	/**
8305	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8306	* stack.
8307	*
8308	* @returns Strict VBox status code (i.e. informational status codes too).
8309	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8310	* @param pVCpu The cross context virtual CPU structure.
8311	* @param uValue The value to push to the guest stack.
8312	*/
8313	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
8314	{
8315	/*
8316	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8317	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8318	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8319	*/
8320	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8321	if (pCtx->sp == 1)
8322	return VINF_EM_RESET;
8323	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8324	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8325	AssertRC(rc);
8326	return rc;
8327	}
8328
8329
8330	/**
8331	* Injects an event into the guest upon VM-entry by updating the relevant fields
8332	* in the VM-entry area in the VMCS.
8333	*
8334	* @returns Strict VBox status code (i.e. informational status codes too).
8335	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8336	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8337	*
8338	* @param pVCpu The cross context virtual CPU structure.
8339	* @param pVmxTransient The VMX-transient structure.
8340	* @param pEvent The event being injected.
8341	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8342	* will be updated if necessary. This cannot not be NULL.
8343	* @param fStepping Whether we're single-stepping guest execution and should
8344	* return VINF_EM_DBG_STEPPED if the event is injected
8345	* directly (registers modified by us, not by hardware on
8346	* VM-entry).
8347	*/
8348	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8349	uint32_t *pfIntrState)
8350	{
8351	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8352	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8353	Assert(pfIntrState);
8354
8355	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8356	uint32_t u32IntInfo = pEvent->u64IntInfo;
8357	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8358	uint32_t const cbInstr = pEvent->cbInstr;
8359	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8360	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8361	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8362
8363	#ifdef VBOX_STRICT
8364	/*
8365	* Validate the error-code-valid bit for hardware exceptions.
8366	* No error codes for exceptions in real-mode.
8367	*
8368	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8369	*/
8370	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8371	&& !CPUMIsGuestInRealModeEx(pCtx))
8372	{
8373	switch (uVector)
8374	{
8375	case X86_XCPT_PF:
8376	case X86_XCPT_DF:
8377	case X86_XCPT_TS:
8378	case X86_XCPT_NP:
8379	case X86_XCPT_SS:
8380	case X86_XCPT_GP:
8381	case X86_XCPT_AC:
8382	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8383	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8384	RT_FALL_THRU();
8385	default:
8386	break;
8387	}
8388	}
8389
8390	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8391	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8392	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8393	#endif
8394
8395	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8396
8397	/*
8398	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8399	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8400	* interrupt handler in the (real-mode) guest.
8401	*
8402	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8403	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8404	*/
8405	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8406	{
8407	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8408	{
8409	/*
8410	* For CPUs with unrestricted guest execution enabled and with the guest
8411	* in real-mode, we must not set the deliver-error-code bit.
8412	*
8413	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8414	*/
8415	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8416	}
8417	else
8418	{
8419	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8420	Assert(PDMVmmDevHeapIsEnabled(pVM));
8421	Assert(pVM->hm.s.vmx.pRealModeTSS);
8422	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8423
8424	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8425	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8426	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8427	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8428	AssertRCReturn(rc2, rc2);
8429
8430	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8431	size_t const cbIdtEntry = sizeof(X86IDTR16);
8432	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8433	{
8434	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8435	if (uVector == X86_XCPT_DF)
8436	return VINF_EM_RESET;
8437
8438	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8439	No error codes for exceptions in real-mode. */
8440	if (uVector == X86_XCPT_GP)
8441	{
8442	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8443	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8444	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8445	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8446	HMEVENT EventXcptDf;
8447	RT_ZERO(EventXcptDf);
8448	EventXcptDf.u64IntInfo = uXcptDfInfo;
8449	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8450	}
8451
8452	/*
8453	* If we're injecting an event with no valid IDT entry, inject a #GP.
8454	* No error codes for exceptions in real-mode.
8455	*
8456	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8457	*/
8458	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8459	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8460	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8461	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8462	HMEVENT EventXcptGp;
8463	RT_ZERO(EventXcptGp);
8464	EventXcptGp.u64IntInfo = uXcptGpInfo;
8465	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8466	}
8467
8468	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8469	uint16_t uGuestIp = pCtx->ip;
8470	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8471	{
8472	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8473	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8474	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8475	}
8476	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8477	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8478
8479	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8480	X86IDTR16 IdtEntry;
8481	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8482	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8483	AssertRCReturn(rc2, rc2);
8484
8485	/* Construct the stack frame for the interrupt/exception handler. */
8486	VBOXSTRICTRC rcStrict;
8487	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8488	if (rcStrict == VINF_SUCCESS)
8489	{
8490	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8491	if (rcStrict == VINF_SUCCESS)
8492	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8493	}
8494
8495	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8496	if (rcStrict == VINF_SUCCESS)
8497	{
8498	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8499	pCtx->rip = IdtEntry.offSel;
8500	pCtx->cs.Sel = IdtEntry.uSel;
8501	pCtx->cs.ValidSel = IdtEntry.uSel;
8502	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8503	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8504	&& uVector == X86_XCPT_PF)
8505	pCtx->cr2 = GCPtrFault;
8506
8507	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8508	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8509	\| HM_CHANGED_GUEST_RSP);
8510
8511	/*
8512	* If we delivered a hardware exception (other than an NMI) and if there was
8513	* block-by-STI in effect, we should clear it.
8514	*/
8515	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8516	{
8517	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8518	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8519	Log4Func(("Clearing inhibition due to STI\n"));
8520	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8521	}
8522
8523	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8524	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8525
8526	/*
8527	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8528	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8529	*/
8530	pVCpu->hm.s.Event.fPending = false;
8531
8532	/*
8533	* If we eventually support nested-guest execution without unrestricted guest execution,
8534	* we should set fInterceptEvents here.
8535	*/
8536	Assert(!pVmxTransient->fIsNestedGuest);
8537
8538	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8539	if (fStepping)
8540	rcStrict = VINF_EM_DBG_STEPPED;
8541	}
8542	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8543	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8544	return rcStrict;
8545	}
8546	}
8547
8548	/*
8549	* Validate.
8550	*/
8551	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8552	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8553
8554	/*
8555	* Inject the event into the VMCS.
8556	*/
8557	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8558	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8559	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8560	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8561	AssertRC(rc);
8562
8563	/*
8564	* Update guest CR2 if this is a page-fault.
8565	*/
8566	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8567	pCtx->cr2 = GCPtrFault;
8568
8569	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8570	return VINF_SUCCESS;
8571	}
8572
8573
8574	/**
8575	* Evaluates the event to be delivered to the guest and sets it as the pending
8576	* event.
8577	*
8578	* @returns Strict VBox status code (i.e. informational status codes too).
8579	* @param pVCpu The cross context virtual CPU structure.
8580	* @param pVmxTransient The VMX-transient structure.
8581	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8582	*/
8583	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8584	{
8585	Assert(pfIntrState);
8586	Assert(!TRPMHasTrap(pVCpu));
8587
8588	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8589	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8590	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8591
8592	/*
8593	* Get the current interruptibility-state of the guest or nested-guest and
8594	* then figure out what needs to be injected.
8595	*/
8596	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8597	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8598	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8599	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8600
8601	/* We don't support block-by-SMI yet.*/
8602	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
8603
8604	/* Block-by-STI must not be set when interrupts are disabled. */
8605	if (fBlockSti)
8606	{
8607	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
8608	Assert(pCtx->eflags.Bits.u1IF);
8609	}
8610
8611	/* Update interruptibility state to the caller. */
8612	*pfIntrState = fIntrState;
8613
8614	/*
8615	* Toggling of interrupt force-flags here is safe since we update TRPM on
8616	* premature exits to ring-3 before executing guest code, see hmR0VmxExitToRing3().
8617	* We must NOT restore these force-flags.
8618	*/
8619
8620	/** @todo SMI. SMIs take priority over NMIs. */
8621
8622	/*
8623	* Check if an NMI is pending and if the guest or nested-guest can receive them.
8624	* NMIs take priority over external interrupts.
8625	*/
8626	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8627	{
8628	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8629	if ( !pVCpu->hm.s.Event.fPending
8630	&& !fBlockNmi
8631	&& !fBlockSti
8632	&& !fBlockMovSS)
8633	{
8634	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8635	if ( fIsNestedGuest
8636	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8637	return IEMExecVmxVmexitXcptNmi(pVCpu);
8638	#endif
8639	hmR0VmxSetPendingXcptNmi(pVCpu);
8640	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8641	Log4Func(("Pending NMI\n"));
8642	}
8643	else if (!fIsNestedGuest)
8644	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8645	/* else: for nested-guests, NMI-window exiting will be picked up when merging VMCS controls. */
8646	}
8647	/*
8648	* Check if an external interrupt (PIC/APIC) is pending and if the guest or nested-guest
8649	* can receive them. Once PDMGetInterrupt() returns a valid interrupt we -must- deliver
8650	* the interrupt. We can no longer re-request it from the APIC.
8651	*/
8652	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8653	&& !pVCpu->hm.s.fSingleInstruction)
8654	{
8655	Assert(!DBGFIsStepping(pVCpu));
8656	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8657	AssertRCReturn(rc, rc);
8658
8659	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8660	if ( !pVCpu->hm.s.Event.fPending
8661	&& !fBlockInt
8662	&& !fBlockSti
8663	&& !fBlockMovSS)
8664	{
8665	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8666	if ( fIsNestedGuest
8667	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8668	&& !CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8669	{
8670	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8671	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8672	return rcStrict;
8673	}
8674	#endif
8675	uint8_t u8Interrupt;
8676	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8677	if (RT_SUCCESS(rc))
8678	{
8679	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8680	if ( fIsNestedGuest
8681	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8682	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8683	{
8684	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8685	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8686	return rcStrict;
8687	}
8688	#endif
8689	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8690	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8691	}
8692	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8693	{
8694	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8695
8696	if ( !fIsNestedGuest
8697	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8698	hmR0VmxApicSetTprThreshold(pVmcsInfo, u8Interrupt >> 4);
8699	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8700
8701	/*
8702	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8703	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8704	* need to re-set this force-flag here.
8705	*/
8706	}
8707	else
8708	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8709	}
8710	else if (!fIsNestedGuest)
8711	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8712	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCS controls. */
8713	}
8714
8715	return VINF_SUCCESS;
8716	}
8717
8718
8719	/**
8720	* Injects any pending events into the guest if the guest is in a state to
8721	* receive them.
8722	*
8723	* @returns Strict VBox status code (i.e. informational status codes too).
8724	* @param pVCpu The cross context virtual CPU structure.
8725	* @param pVmxTransient The VMX-transient structure.
8726	* @param fIntrState The VT-x guest-interruptibility state.
8727	* @param fStepping Whether we are single-stepping the guest using the
8728	* hypervisor debugger and should return
8729	* VINF_EM_DBG_STEPPED if the event was dispatched
8730	* directly.
8731	*/
8732	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8733	{
8734	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8735	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8736
8737	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8738	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8739
8740	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8741	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8742	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8743	Assert(!TRPMHasTrap(pVCpu));
8744
8745	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8746	if (pVCpu->hm.s.Event.fPending)
8747	{
8748	/*
8749	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8750	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8751	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8752	*
8753	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8754	*/
8755	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8756	#ifdef VBOX_STRICT
8757	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8758	{
8759	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8760	Assert(!fBlockInt);
8761	Assert(!fBlockSti);
8762	Assert(!fBlockMovSS);
8763	}
8764	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8765	{
8766	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8767	Assert(!fBlockSti);
8768	Assert(!fBlockMovSS);
8769	Assert(!fBlockNmi);
8770	}
8771	#endif
8772	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8773	uIntType));
8774
8775	/*
8776	* Inject the event and get any changes to the guest-interruptibility state.
8777	*
8778	* The guest-interruptibility state may need to be updated if we inject the event
8779	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8780	*/
8781	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8782	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8783
8784	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8785	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8786	else
8787	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8788	}
8789
8790	/*
8791	* Update the guest-interruptibility state.
8792	*
8793	* This is required for the real-on-v86 software interrupt injection case above, as well as
8794	* updates to the guest state from ring-3 or IEM/REM.
8795	*/
8796	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8797	AssertRC(rc);
8798
8799	/*
8800	* There's no need to clear the VM-entry interruption-information field here if we're not
8801	* injecting anything. VT-x clears the valid bit on every VM-exit.
8802	*
8803	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8804	*/
8805
8806	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8807	NOREF(fBlockMovSS); NOREF(fBlockSti);
8808	return rcStrict;
8809	}
8810
8811
8812	/**
8813	* Enters the VT-x session.
8814	*
8815	* @returns VBox status code.
8816	* @param pVCpu The cross context virtual CPU structure.
8817	*/
8818	VMMR0DECL(int) VMXR0Enter(PVMCPUCC pVCpu)
8819	{
8820	AssertPtr(pVCpu);
8821	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8822	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8823
8824	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8825	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8826	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8827
8828	#ifdef VBOX_STRICT
8829	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8830	RTCCUINTREG uHostCr4 = ASMGetCR4();
8831	if (!(uHostCr4 & X86_CR4_VMXE))
8832	{
8833	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8834	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8835	}
8836	#endif
8837
8838	/*
8839	* Load the appropriate VMCS as the current and active one.
8840	*/
8841	PVMXVMCSINFO pVmcsInfo;
8842	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8843	if (!fInNestedGuestMode)
8844	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8845	else
8846	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8847	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8848	if (RT_SUCCESS(rc))
8849	{
8850	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8851	pVCpu->hm.s.fLeaveDone = false;
8852	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8853
8854	/*
8855	* Do the EMT scheduled L1D flush here if needed.
8856	*/
8857	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8858	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8859	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8860	hmR0MdsClear();
8861	}
8862	return rc;
8863	}
8864
8865
8866	/**
8867	* The thread-context callback (only on platforms which support it).
8868	*
8869	* @param enmEvent The thread-context event.
8870	* @param pVCpu The cross context virtual CPU structure.
8871	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8872	* @thread EMT(pVCpu)
8873	*/
8874	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
8875	{
8876	AssertPtr(pVCpu);
8877	RT_NOREF1(fGlobalInit);
8878
8879	switch (enmEvent)
8880	{
8881	case RTTHREADCTXEVENT_OUT:
8882	{
8883	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8884	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8885	VMCPU_ASSERT_EMT(pVCpu);
8886
8887	/* No longjmps (logger flushes, locks) in this fragile context. */
8888	VMMRZCallRing3Disable(pVCpu);
8889	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8890
8891	/* Restore host-state (FPU, debug etc.) */
8892	if (!pVCpu->hm.s.fLeaveDone)
8893	{
8894	/*
8895	* Do -not- import the guest-state here as we might already be in the middle of importing
8896	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8897	*/
8898	hmR0VmxLeave(pVCpu, false /* fImportState */);
8899	pVCpu->hm.s.fLeaveDone = true;
8900	}
8901
8902	/* Leave HM context, takes care of local init (term). */
8903	int rc = HMR0LeaveCpu(pVCpu);
8904	AssertRC(rc);
8905
8906	/* Restore longjmp state. */
8907	VMMRZCallRing3Enable(pVCpu);
8908	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8909	break;
8910	}
8911
8912	case RTTHREADCTXEVENT_IN:
8913	{
8914	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8915	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8916	VMCPU_ASSERT_EMT(pVCpu);
8917
8918	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8919	VMMRZCallRing3Disable(pVCpu);
8920	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8921
8922	/* Initialize the bare minimum state required for HM. This takes care of
8923	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8924	int rc = hmR0EnterCpu(pVCpu);
8925	AssertRC(rc);
8926	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8927	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8928
8929	/* Load the active VMCS as the current one. */
8930	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8931	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8932	AssertRC(rc);
8933	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8934	pVCpu->hm.s.fLeaveDone = false;
8935
8936	/* Do the EMT scheduled L1D flush if needed. */
8937	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8938	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8939
8940	/* Restore longjmp state. */
8941	VMMRZCallRing3Enable(pVCpu);
8942	break;
8943	}
8944
8945	default:
8946	break;
8947	}
8948	}
8949
8950
8951	/**
8952	* Exports the host state into the VMCS host-state area.
8953	* Sets up the VM-exit MSR-load area.
8954	*
8955	* The CPU state will be loaded from these fields on every successful VM-exit.
8956	*
8957	* @returns VBox status code.
8958	* @param pVCpu The cross context virtual CPU structure.
8959	*
8960	* @remarks No-long-jump zone!!!
8961	*/
8962	static int hmR0VmxExportHostState(PVMCPUCC pVCpu)
8963	{
8964	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8965
8966	int rc = VINF_SUCCESS;
8967	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8968	{
8969	hmR0VmxExportHostControlRegs();
8970
8971	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8972	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8973
8974	hmR0VmxExportHostMsrs(pVCpu);
8975
8976	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8977	}
8978	return rc;
8979	}
8980
8981
8982	/**
8983	* Saves the host state in the VMCS host-state.
8984	*
8985	* @returns VBox status code.
8986	* @param pVCpu The cross context virtual CPU structure.
8987	*
8988	* @remarks No-long-jump zone!!!
8989	*/
8990	VMMR0DECL(int) VMXR0ExportHostState(PVMCPUCC pVCpu)
8991	{
8992	AssertPtr(pVCpu);
8993	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8994
8995	/*
8996	* Export the host state here while entering HM context.
8997	* When thread-context hooks are used, we might get preempted and have to re-save the host
8998	* state but most of the time we won't be, so do it here before we disable interrupts.
8999	*/
9000	return hmR0VmxExportHostState(pVCpu);
9001	}
9002
9003
9004	/**
9005	* Exports the guest state into the VMCS guest-state area.
9006	*
9007	* The will typically be done before VM-entry when the guest-CPU state and the
9008	* VMCS state may potentially be out of sync.
9009	*
9010	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9011	* VM-entry controls.
9012	* Sets up the appropriate VMX non-root function to execute guest code based on
9013	* the guest CPU mode.
9014	*
9015	* @returns VBox strict status code.
9016	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9017	* without unrestricted guest execution and the VMMDev is not presently
9018	* mapped (e.g. EFI32).
9019	*
9020	* @param pVCpu The cross context virtual CPU structure.
9021	* @param pVmxTransient The VMX-transient structure.
9022	*
9023	* @remarks No-long-jump zone!!!
9024	*/
9025	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9026	{
9027	AssertPtr(pVCpu);
9028	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9029	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9030
9031	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9032
9033	/*
9034	* Determine real-on-v86 mode.
9035	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9036	*/
9037	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9038	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9039	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9040	pVmcsInfo->RealMode. fRealOnV86Active = false;
9041	else
9042	{
9043	Assert(!pVmxTransient->fIsNestedGuest);
9044	pVmcsInfo->RealMode.fRealOnV86Active = true;
9045	}
9046
9047	/*
9048	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9049	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9050	*/
9051	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9052	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9053	* be a need to evaluate this everytime since I'm pretty sure we intercept
9054	* all guest paging mode changes. */
9055	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9056	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9057
9058	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9059	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9060
9061	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9062	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9063
9064	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9065	if (rcStrict == VINF_SUCCESS)
9066	{ /* likely */ }
9067	else
9068	{
9069	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9070	return rcStrict;
9071	}
9072
9073	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9074	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9075
9076	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9077	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9078
9079	hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9080	hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9081	hmR0VmxExportGuestRip(pVCpu);
9082	hmR0VmxExportGuestRsp(pVCpu);
9083	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9084
9085	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9086	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9087
9088	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9089	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9090	\| HM_CHANGED_GUEST_CR2
9091	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9092	\| HM_CHANGED_GUEST_X87
9093	\| HM_CHANGED_GUEST_SSE_AVX
9094	\| HM_CHANGED_GUEST_OTHER_XSAVE
9095	\| HM_CHANGED_GUEST_XCRx
9096	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9097	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9098	\| HM_CHANGED_GUEST_TSC_AUX
9099	\| HM_CHANGED_GUEST_OTHER_MSRS
9100	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9101
9102	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9103	return rc;
9104	}
9105
9106
9107	/**
9108	* Exports the state shared between the host and guest into the VMCS.
9109	*
9110	* @param pVCpu The cross context virtual CPU structure.
9111	* @param pVmxTransient The VMX-transient structure.
9112	*
9113	* @remarks No-long-jump zone!!!
9114	*/
9115	static void hmR0VmxExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9116	{
9117	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9118	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9119
9120	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9121	{
9122	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9123	AssertRC(rc);
9124	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9125
9126	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9127	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9128	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9129	}
9130
9131	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9132	{
9133	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9134	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9135	}
9136
9137	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9138	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9139	}
9140
9141
9142	/**
9143	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9144	*
9145	* @returns Strict VBox status code (i.e. informational status codes too).
9146	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9147	* without unrestricted guest execution and the VMMDev is not presently
9148	* mapped (e.g. EFI32).
9149	*
9150	* @param pVCpu The cross context virtual CPU structure.
9151	* @param pVmxTransient The VMX-transient structure.
9152	*
9153	* @remarks No-long-jump zone!!!
9154	*/
9155	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9156	{
9157	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9158	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9159	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9160
9161	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9162	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9163	#endif
9164
9165	/*
9166	* For many exits it's only RIP/RSP/RFLAGS that changes and hence try to export it first
9167	* without going through a lot of change flag checks.
9168	*/
9169	VBOXSTRICTRC rcStrict;
9170	uint64_t const fCtxMask = HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
9171	uint64_t const fMinimalMask = HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT;
9172
9173	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9174	/* If only RIP/RSP/RFLAGS/HWVIRT changed, export only those (quicker, happens more often).*/
9175	if ( (fCtxChanged & fMinimalMask)
9176	&& !(fCtxChanged & (fCtxMask & ~fMinimalMask)))
9177	{
9178	hmR0VmxExportGuestRip(pVCpu);
9179	hmR0VmxExportGuestRsp(pVCpu);
9180	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9181	rcStrict = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9182	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9183	}
9184	/* If anything else also changed, go through the full export routine and export as required. */
9185	else if (fCtxChanged & fCtxMask)
9186	{
9187	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9188	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9189	{ /* likely */}
9190	else
9191	{
9192	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9193	VBOXSTRICTRC_VAL(rcStrict)));
9194	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9195	return rcStrict;
9196	}
9197	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9198	}
9199	/* Nothing changed, nothing to load here. */
9200	else
9201	rcStrict = VINF_SUCCESS;
9202
9203	#ifdef VBOX_STRICT
9204	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9205	uint64_t const fCtxChangedCur = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9206	AssertMsg(!(fCtxChangedCur & fCtxMask), ("fCtxChangedCur=%#RX64\n", fCtxChangedCur));
9207	#endif
9208	return rcStrict;
9209	}
9210
9211
9212	/**
9213	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9214	* and update error record fields accordingly.
9215	*
9216	* @returns VMX_IGS_* error codes.
9217	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9218	* wrong with the guest state.
9219	*
9220	* @param pVCpu The cross context virtual CPU structure.
9221	* @param pVmcsInfo The VMCS info. object.
9222	*
9223	* @remarks This function assumes our cache of the VMCS controls
9224	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9225	*/
9226	static uint32_t hmR0VmxCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9227	{
9228	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9229	#define HMVMX_CHECK_BREAK(expr, err) do { \
9230	if (!(expr)) { uError = (err); break; } \
9231	} while (0)
9232
9233	int rc;
9234	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9235	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9236	uint32_t uError = VMX_IGS_ERROR;
9237	uint32_t u32Val;
9238	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9239
9240	do
9241	{
9242	/*
9243	* CR0.
9244	*/
9245	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9246	uint64_t fSetCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9247	uint64_t const fZapCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9248	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
9249	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9250	if (fUnrestrictedGuest)
9251	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
9252
9253	uint64_t u64GuestCr0;
9254	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64GuestCr0);
9255	AssertRC(rc);
9256	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9257	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9258	if ( !fUnrestrictedGuest
9259	&& (u64GuestCr0 & X86_CR0_PG)
9260	&& !(u64GuestCr0 & X86_CR0_PE))
9261	{
9262	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9263	}
9264
9265	/*
9266	* CR4.
9267	*/
9268	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9269	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9270	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9271
9272	uint64_t u64GuestCr4;
9273	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64GuestCr4);
9274	AssertRC(rc);
9275	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9276	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9277
9278	/*
9279	* IA32_DEBUGCTL MSR.
9280	*/
9281	uint64_t u64Val;
9282	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9283	AssertRC(rc);
9284	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9285	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9286	{
9287	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9288	}
9289	uint64_t u64DebugCtlMsr = u64Val;
9290
9291	#ifdef VBOX_STRICT
9292	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9293	AssertRC(rc);
9294	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9295	#endif
9296	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9297
9298	/*
9299	* RIP and RFLAGS.
9300	*/
9301	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
9302	AssertRC(rc);
9303	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9304	if ( !fLongModeGuest
9305	\|\| !pCtx->cs.Attr.n.u1Long)
9306	{
9307	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9308	}
9309	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9310	* must be identical if the "IA-32e mode guest" VM-entry
9311	* control is 1 and CS.L is 1. No check applies if the
9312	* CPU supports 64 linear-address bits. */
9313
9314	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9315	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9316	AssertRC(rc);
9317	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9318	VMX_IGS_RFLAGS_RESERVED);
9319	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9320	uint32_t const u32Eflags = u64Val;
9321
9322	if ( fLongModeGuest
9323	\|\| ( fUnrestrictedGuest
9324	&& !(u64GuestCr0 & X86_CR0_PE)))
9325	{
9326	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9327	}
9328
9329	uint32_t u32EntryInfo;
9330	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9331	AssertRC(rc);
9332	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9333	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9334
9335	/*
9336	* 64-bit checks.
9337	*/
9338	if (fLongModeGuest)
9339	{
9340	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9341	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9342	}
9343
9344	if ( !fLongModeGuest
9345	&& (u64GuestCr4 & X86_CR4_PCIDE))
9346	{
9347	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9348	}
9349
9350	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9351	* 51:32 beyond the processor's physical-address width are 0. */
9352
9353	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9354	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9355	{
9356	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9357	}
9358
9359	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9360	AssertRC(rc);
9361	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9362
9363	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9364	AssertRC(rc);
9365	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9366
9367	/*
9368	* PERF_GLOBAL MSR.
9369	*/
9370	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9371	{
9372	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9373	AssertRC(rc);
9374	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9375	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9376	}
9377
9378	/*
9379	* PAT MSR.
9380	*/
9381	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9382	{
9383	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9384	AssertRC(rc);
9385	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9386	for (unsigned i = 0; i < 8; i++)
9387	{
9388	uint8_t u8Val = (u64Val & 0xff);
9389	if ( u8Val != 0 /* UC */
9390	&& u8Val != 1 /* WC */
9391	&& u8Val != 4 /* WT */
9392	&& u8Val != 5 /* WP */
9393	&& u8Val != 6 /* WB */
9394	&& u8Val != 7 /* UC- */)
9395	{
9396	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9397	}
9398	u64Val >>= 8;
9399	}
9400	}
9401
9402	/*
9403	* EFER MSR.
9404	*/
9405	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9406	{
9407	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9408	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9409	AssertRC(rc);
9410	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9411	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9412	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9413	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9414	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9415	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9416	* iemVmxVmentryCheckGuestState(). */
9417	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9418	\|\| !(u64GuestCr0 & X86_CR0_PG)
9419	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9420	VMX_IGS_EFER_LMA_LME_MISMATCH);
9421	}
9422
9423	/*
9424	* Segment registers.
9425	*/
9426	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9427	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9428	if (!(u32Eflags & X86_EFL_VM))
9429	{
9430	/* CS */
9431	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9432	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9433	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9434	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9435	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9436	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9437	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9438	/* CS cannot be loaded with NULL in protected mode. */
9439	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9440	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9441	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9442	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9443	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9444	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9445	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9446	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9447	else
9448	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9449
9450	/* SS */
9451	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9452	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9453	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9454	if ( !(pCtx->cr0 & X86_CR0_PE)
9455	\|\| pCtx->cs.Attr.n.u4Type == 3)
9456	{
9457	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9458	}
9459	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9460	{
9461	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9462	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9463	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9464	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9465	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9466	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9467	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9468	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9469	}
9470
9471	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9472	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9473	{
9474	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9475	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9476	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9477	\|\| pCtx->ds.Attr.n.u4Type > 11
9478	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9479	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9480	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9481	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9482	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9483	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9484	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9485	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9486	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9487	}
9488	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9489	{
9490	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9491	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9492	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9493	\|\| pCtx->es.Attr.n.u4Type > 11
9494	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9495	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9496	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9497	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9498	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9499	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9500	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9501	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9502	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9503	}
9504	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9505	{
9506	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9507	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9508	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9509	\|\| pCtx->fs.Attr.n.u4Type > 11
9510	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9511	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9512	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9513	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9514	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9515	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9516	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9517	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9518	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9519	}
9520	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9521	{
9522	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9523	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9524	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9525	\|\| pCtx->gs.Attr.n.u4Type > 11
9526	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9527	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9528	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9529	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9530	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9531	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9532	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9533	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9534	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9535	}
9536	/* 64-bit capable CPUs. */
9537	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9538	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9539	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9540	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9541	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9542	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9543	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9544	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9545	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9546	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9547	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9548	}
9549	else
9550	{
9551	/* V86 mode checks. */
9552	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9553	if (pVmcsInfo->RealMode.fRealOnV86Active)
9554	{
9555	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9556	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9557	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9558	}
9559	else
9560	{
9561	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9562	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9563	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9564	}
9565
9566	/* CS */
9567	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9568	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9569	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9570	/* SS */
9571	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9572	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9573	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9574	/* DS */
9575	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9576	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9577	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9578	/* ES */
9579	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9580	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9581	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9582	/* FS */
9583	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9584	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9585	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9586	/* GS */
9587	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9588	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9589	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9590	/* 64-bit capable CPUs. */
9591	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9592	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9593	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9594	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9595	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9596	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9597	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9598	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9599	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9600	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9601	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9602	}
9603
9604	/*
9605	* TR.
9606	*/
9607	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9608	/* 64-bit capable CPUs. */
9609	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9610	if (fLongModeGuest)
9611	{
9612	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9613	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9614	}
9615	else
9616	{
9617	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9618	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9619	VMX_IGS_TR_ATTR_TYPE_INVALID);
9620	}
9621	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9622	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9623	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9624	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9625	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9626	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9627	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9628	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9629
9630	/*
9631	* GDTR and IDTR (64-bit capable checks).
9632	*/
9633	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9634	AssertRC(rc);
9635	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9636
9637	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9638	AssertRC(rc);
9639	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9640
9641	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9642	AssertRC(rc);
9643	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9644
9645	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9646	AssertRC(rc);
9647	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9648
9649	/*
9650	* Guest Non-Register State.
9651	*/
9652	/* Activity State. */
9653	uint32_t u32ActivityState;
9654	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9655	AssertRC(rc);
9656	HMVMX_CHECK_BREAK( !u32ActivityState
9657	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9658	VMX_IGS_ACTIVITY_STATE_INVALID);
9659	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9660	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9661	uint32_t u32IntrState;
9662	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9663	AssertRC(rc);
9664	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9665	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9666	{
9667	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9668	}
9669
9670	/** @todo Activity state and injecting interrupts. Left as a todo since we
9671	* currently don't use activity states but ACTIVE. */
9672
9673	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9674	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9675
9676	/* Guest interruptibility-state. */
9677	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9678	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9679	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9680	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9681	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9682	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9683	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9684	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9685	{
9686	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9687	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9688	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9689	}
9690	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9691	{
9692	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9693	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9694	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9695	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9696	}
9697	/** @todo Assumes the processor is not in SMM. */
9698	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9699	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9700	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9701	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9702	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9703	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9704	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9705	{
9706	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9707	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9708	}
9709
9710	/* Pending debug exceptions. */
9711	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9712	AssertRC(rc);
9713	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9714	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9715	u32Val = u64Val; /* For pending debug exceptions checks below. */
9716
9717	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9718	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9719	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9720	{
9721	if ( (u32Eflags & X86_EFL_TF)
9722	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9723	{
9724	/* Bit 14 is PendingDebug.BS. */
9725	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9726	}
9727	if ( !(u32Eflags & X86_EFL_TF)
9728	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9729	{
9730	/* Bit 14 is PendingDebug.BS. */
9731	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9732	}
9733	}
9734
9735	/* VMCS link pointer. */
9736	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9737	AssertRC(rc);
9738	if (u64Val != UINT64_C(0xffffffffffffffff))
9739	{
9740	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9741	/** @todo Bits beyond the processor's physical-address width MBZ. */
9742	/** @todo SMM checks. */
9743	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
9744	Assert(pVmcsInfo->pvShadowVmcs);
9745	VMXVMCSREVID VmcsRevId;
9746	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
9747	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
9748	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
9749	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
9750	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
9751	}
9752
9753	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9754	* not using nested paging? */
9755	if ( pVM->hm.s.fNestedPaging
9756	&& !fLongModeGuest
9757	&& CPUMIsGuestInPAEModeEx(pCtx))
9758	{
9759	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9760	AssertRC(rc);
9761	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9762
9763	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9764	AssertRC(rc);
9765	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9766
9767	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9768	AssertRC(rc);
9769	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9770
9771	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9772	AssertRC(rc);
9773	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9774	}
9775
9776	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9777	if (uError == VMX_IGS_ERROR)
9778	uError = VMX_IGS_REASON_NOT_FOUND;
9779	} while (0);
9780
9781	pVCpu->hm.s.u32HMError = uError;
9782	return uError;
9783
9784	#undef HMVMX_ERROR_BREAK
9785	#undef HMVMX_CHECK_BREAK
9786	}
9787
9788
9789	/**
9790	* Map the APIC-access page for virtualizing APIC accesses.
9791	*
9792	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
9793	* this not done as part of exporting guest state, see @bugref{8721}.
9794	*
9795	* @returns VBox status code.
9796	* @param pVCpu The cross context virtual CPU structure.
9797	*/
9798	static int hmR0VmxMapHCApicAccessPage(PVMCPUCC pVCpu)
9799	{
9800	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9801	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9802
9803	Assert(PDMHasApic(pVM));
9804	Assert(u64MsrApicBase);
9805
9806	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9807	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9808
9809	/* Unalias the existing mapping. */
9810	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9811	AssertRCReturn(rc, rc);
9812
9813	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9814	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9815	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9816	AssertRCReturn(rc, rc);
9817
9818	/* Update the per-VCPU cache of the APIC base MSR. */
9819	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9820	return VINF_SUCCESS;
9821	}
9822
9823
9824	/**
9825	* Worker function passed to RTMpOnSpecific() that is to be called on the target
9826	* CPU.
9827	*
9828	* @param idCpu The ID for the CPU the function is called on.
9829	* @param pvUser1 Null, not used.
9830	* @param pvUser2 Null, not used.
9831	*/
9832	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
9833	{
9834	RT_NOREF3(idCpu, pvUser1, pvUser2);
9835	VMXDispatchHostNmi();
9836	}
9837
9838
9839	/**
9840	* Dispatching an NMI on the host CPU that received it.
9841	*
9842	* @returns VBox status code.
9843	* @param pVCpu The cross context virtual CPU structure.
9844	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
9845	* executing when receiving the host NMI in VMX non-root
9846	* operation.
9847	*/
9848	static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9849	{
9850	RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
9851	Assert(idCpu != NIL_RTCPUID);
9852
9853	/*
9854	* We don't want to delay dispatching the NMI any more than we have to. However,
9855	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
9856	* after executing guest or nested-guest code for the following reasons:
9857	*
9858	* - We would need to perform VMREADs with interrupts disabled and is orders of
9859	* magnitude worse when we run as a nested hypervisor without VMCS shadowing
9860	* supported by the host hypervisor.
9861	*
9862	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
9863	* longer period of time just for handling an edge case like host NMIs which do
9864	* not occur nearly as frequently as other VM-exits.
9865	*
9866	* Let's cover the most likely scenario first. Check if we are on the target CPU
9867	* and dispatch the NMI right away. This should be much faster than calling into
9868	* RTMpOnSpecific() machinery.
9869	*/
9870	bool fDispatched = false;
9871	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
9872	if (idCpu == RTMpCpuId())
9873	{
9874	VMXDispatchHostNmi();
9875	fDispatched = true;
9876	}
9877	ASMSetFlags(fEFlags);
9878	if (fDispatched)
9879	{
9880	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9881	return VINF_SUCCESS;
9882	}
9883
9884	/*
9885	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
9886	* there should be no race or recursion even if we are unlucky enough to be preempted
9887	* (to the target CPU) without dispatching the host NMI above.
9888	*/
9889	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
9890	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
9891	}
9892
9893
9894	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9895	/**
9896	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9897	* nested-guest using hardware-assisted VMX.
9898	*
9899	* @param pVCpu The cross context virtual CPU structure.
9900	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9901	* @param pVmcsInfoGst The guest VMCS info. object.
9902	*/
9903	static void hmR0VmxMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9904	{
9905	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
9906	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9907	Assert(pu64MsrBitmap);
9908
9909	/*
9910	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9911	* MSR that is intercepted by the guest is also intercepted while executing the
9912	* nested-guest using hardware-assisted VMX.
9913	*
9914	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
9915	* nested-guest VM-exit even if the outer guest is not intercepting some
9916	* MSRs. We cannot assume the caller has initialized the nested-guest
9917	* MSR bitmap in this case.
9918	*
9919	* The nested hypervisor may also switch whether it uses MSR bitmaps for
9920	* each VM-entry, hence initializing it once per-VM while setting up the
9921	* nested-guest VMCS is not sufficient.
9922	*/
9923	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9924	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
9925	{
9926	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9927	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9928	Assert(pu64MsrBitmapNstGst);
9929	Assert(pu64MsrBitmapGst);
9930
9931	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
9932	for (uint32_t i = 0; i < cFrags; i++)
9933	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9934	}
9935	else
9936	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
9937	}
9938
9939
9940	/**
9941	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9942	* hardware-assisted VMX execution of the nested-guest.
9943	*
9944	* For a guest, we don't modify these controls once we set up the VMCS and hence
9945	* this function is never called.
9946	*
9947	* For nested-guests since the nested hypervisor provides these controls on every
9948	* nested-guest VM-entry and could potentially change them everytime we need to
9949	* merge them before every nested-guest VM-entry.
9950	*
9951	* @returns VBox status code.
9952	* @param pVCpu The cross context virtual CPU structure.
9953	*/
9954	static int hmR0VmxMergeVmcsNested(PVMCPUCC pVCpu)
9955	{
9956	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9957	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9958	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9959	Assert(pVmcsNstGst);
9960
9961	/*
9962	* Merge the controls with the requirements of the guest VMCS.
9963	*
9964	* We do not need to validate the nested-guest VMX features specified in the nested-guest
9965	* VMCS with the features supported by the physical CPU as it's already done by the
9966	* VMLAUNCH/VMRESUME instruction emulation.
9967	*
9968	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
9969	* derived from the VMX features supported by the physical CPU.
9970	*/
9971
9972	/* Pin-based VM-execution controls. */
9973	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9974
9975	/* Processor-based VM-execution controls. */
9976	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9977	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9978	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9979	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9980	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9981
9982	/* Secondary processor-based VM-execution controls. */
9983	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9984	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9985	\| VMX_PROC_CTLS2_INVPCID
9986	\| VMX_PROC_CTLS2_VMCS_SHADOWING
9987	\| VMX_PROC_CTLS2_RDTSCP
9988	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9989	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9990	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9991	\| VMX_PROC_CTLS2_VMFUNC));
9992
9993	/*
9994	* VM-entry controls:
9995	* These controls contains state that depends on the nested-guest state (primarily
9996	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
9997	* VM-exit. Although the nested hypervisor cannot change it, we need to in order to
9998	* properly continue executing the nested-guest if the EFER MSR changes but does not
9999	* cause a nested-guest VM-exits.
10000	*
10001	* VM-exit controls:
10002	* These controls specify the host state on return. We cannot use the controls from
10003	* the nested hypervisor state as is as it would contain the guest state rather than
10004	* the host state. Since the host state is subject to change (e.g. preemption, trips
10005	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10006	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10007	*
10008	* VM-entry MSR-load:
10009	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10010	* context by the VMLAUNCH/VMRESUME instruction emulation.
10011	*
10012	* VM-exit MSR-store:
10013	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10014	* back into the VM-exit MSR-store area.
10015	*
10016	* VM-exit MSR-load areas:
10017	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10018	* can entirely ignore what the nested hypervisor wants to load here.
10019	*/
10020
10021	/*
10022	* Exception bitmap.
10023	*
10024	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10025	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10026	* code more flexible if intercepting exceptions become more dynamic in the future we do
10027	* it as part of exporting the nested-guest state.
10028	*/
10029	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10030
10031	/*
10032	* CR0/CR4 guest/host mask.
10033	*
10034	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10035	* cause VM-exits, so we need to merge them here.
10036	*/
10037	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10038	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10039
10040	/*
10041	* Page-fault error-code mask and match.
10042	*
10043	* Although we require unrestricted guest execution (and thereby nested-paging) for
10044	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10045	* normally intercept #PFs, it might intercept them for debugging purposes.
10046	*
10047	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10048	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10049	*/
10050	uint32_t u32XcptPFMask;
10051	uint32_t u32XcptPFMatch;
10052	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10053	{
10054	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10055	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10056	}
10057	else
10058	{
10059	u32XcptPFMask = 0;
10060	u32XcptPFMatch = 0;
10061	}
10062
10063	/*
10064	* Pause-Loop exiting.
10065	*/
10066	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10067	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10068
10069	/*
10070	* Pending debug exceptions.
10071	* Currently just copy whatever the nested-guest provides us.
10072	*/
10073	uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
10074
10075	/*
10076	* I/O Bitmap.
10077	*
10078	* We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
10079	* intercept all I/O port accesses.
10080	*/
10081	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10082	Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
10083
10084	/*
10085	* VMCS shadowing.
10086	*
10087	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10088	* enabled while executing the nested-guest.
10089	*/
10090	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10091
10092	/*
10093	* APIC-access page.
10094	*/
10095	RTHCPHYS HCPhysApicAccess;
10096	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10097	{
10098	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10099	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10100
10101	/** @todo NSTVMX: This is not really correct but currently is required to make
10102	* things work. We need to re-enable the page handler when we fallback to
10103	* IEM execution of the nested-guest! */
10104	PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
10105
10106	void *pvPage;
10107	PGMPAGEMAPLOCK PgLockApicAccess;
10108	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
10109	if (RT_SUCCESS(rc))
10110	{
10111	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10112	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10113
10114	/** @todo Handle proper releasing of page-mapping lock later. */
10115	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
10116	}
10117	else
10118	return rc;
10119	}
10120	else
10121	HCPhysApicAccess = 0;
10122
10123	/*
10124	* Virtual-APIC page and TPR threshold.
10125	*/
10126	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10127	RTHCPHYS HCPhysVirtApic;
10128	uint32_t u32TprThreshold;
10129	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10130	{
10131	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10132	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10133
10134	void *pvPage;
10135	PGMPAGEMAPLOCK PgLockVirtApic;
10136	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
10137	if (RT_SUCCESS(rc))
10138	{
10139	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10140	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10141
10142	/** @todo Handle proper releasing of page-mapping lock later. */
10143	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
10144	}
10145	else
10146	return rc;
10147
10148	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10149	}
10150	else
10151	{
10152	HCPhysVirtApic = 0;
10153	u32TprThreshold = 0;
10154
10155	/*
10156	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10157	* used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
10158	* be taken care of by EPT/shadow paging.
10159	*/
10160	if (pVM->hm.s.fAllow64BitGuests)
10161	{
10162	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10163	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10164	}
10165	}
10166
10167	/*
10168	* Validate basic assumptions.
10169	*/
10170	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10171	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10172	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10173
10174	/*
10175	* Commit it to the nested-guest VMCS.
10176	*/
10177	int rc = VINF_SUCCESS;
10178	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10179	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10180	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10181	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10182	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10183	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10184	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10185	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10186	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10187	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10188	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10189	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10190	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10191	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10192	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10193	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10194	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10195	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10196	{
10197	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10198	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10199	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10200	}
10201	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10202	{
10203	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10204	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10205	}
10206	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10207	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10208	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
10209	AssertRC(rc);
10210
10211	/*
10212	* Update the nested-guest VMCS cache.
10213	*/
10214	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10215	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10216	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10217	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10218	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10219	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10220	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10221	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10222	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10223
10224	/*
10225	* We need to flush the TLB if we are switching the APIC-access page address.
10226	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10227	*/
10228	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10229	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10230
10231	/*
10232	* MSR bitmap.
10233	*
10234	* The MSR bitmap address has already been initialized while setting up the nested-guest
10235	* VMCS, here we need to merge the MSR bitmaps.
10236	*/
10237	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10238	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10239
10240	return VINF_SUCCESS;
10241	}
10242	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10243
10244
10245	/**
10246	* Does the preparations before executing guest code in VT-x.
10247	*
10248	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10249	* recompiler/IEM. We must be cautious what we do here regarding committing
10250	* guest-state information into the VMCS assuming we assuredly execute the
10251	* guest in VT-x mode.
10252	*
10253	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10254	* the common-state (TRPM/forceflags), we must undo those changes so that the
10255	* recompiler/IEM can (and should) use them when it resumes guest execution.
10256	* Otherwise such operations must be done when we can no longer exit to ring-3.
10257	*
10258	* @returns Strict VBox status code (i.e. informational status codes too).
10259	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10260	* have been disabled.
10261	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10262	* pending events).
10263	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10264	* double-fault into the guest.
10265	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10266	* dispatched directly.
10267	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10268	*
10269	* @param pVCpu The cross context virtual CPU structure.
10270	* @param pVmxTransient The VMX-transient structure.
10271	* @param fStepping Whether we are single-stepping the guest in the
10272	* hypervisor debugger. Makes us ignore some of the reasons
10273	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10274	* if event dispatching took place.
10275	*/
10276	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10277	{
10278	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10279
10280	Log4Func(("fIsNested=%RTbool fStepping=%RTbool\n", pVmxTransient->fIsNestedGuest, fStepping));
10281
10282	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10283	if (pVmxTransient->fIsNestedGuest)
10284	{
10285	RT_NOREF2(pVCpu, fStepping);
10286	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10287	return VINF_EM_RESCHEDULE_REM;
10288	}
10289	#endif
10290
10291	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10292	PGMRZDynMapFlushAutoSet(pVCpu);
10293	#endif
10294
10295	/*
10296	* Check and process force flag actions, some of which might require us to go back to ring-3.
10297	*/
10298	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10299	if (rcStrict == VINF_SUCCESS)
10300	{
10301	/* FFs don't get set all the time. */
10302	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10303	if ( pVmxTransient->fIsNestedGuest
10304	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10305	{
10306	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10307	return VINF_VMX_VMEXIT;
10308	}
10309	#endif
10310	}
10311	else
10312	return rcStrict;
10313
10314	/*
10315	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10316	*/
10317	/** @todo Doing this from ring-3 after VM setup phase causes a
10318	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
10319	* idea why atm. */
10320	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10321	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10322	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10323	&& PDMHasApic(pVM))
10324	{
10325	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10326	AssertRCReturn(rc, rc);
10327	}
10328
10329	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10330	/*
10331	* Merge guest VMCS controls with the nested-guest VMCS controls.
10332	*
10333	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10334	* saved state), we should be okay with merging controls as we initialize the
10335	* guest VMCS controls as part of VM setup phase.
10336	*/
10337	if ( pVmxTransient->fIsNestedGuest
10338	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10339	{
10340	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10341	AssertRCReturn(rc, rc);
10342	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10343	}
10344	#endif
10345
10346	/*
10347	* Evaluate events to be injected into the guest.
10348	*
10349	* Events in TRPM can be injected without inspecting the guest state.
10350	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10351	* guest to cause a VM-exit the next time they are ready to receive the event.
10352	*
10353	* With nested-guests, evaluating pending events may cause VM-exits. Also, verify
10354	* that the event in TRPM that we will inject using hardware-assisted VMX is -not-
10355	* subject to interecption. Otherwise, we should have checked and injected them
10356	* manually elsewhere (IEM).
10357	*/
10358	if (TRPMHasTrap(pVCpu))
10359	{
10360	Assert(!pVmxTransient->fIsNestedGuest \|\| !CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
10361	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10362	}
10363
10364	uint32_t fIntrState;
10365	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10366
10367	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10368	/*
10369	* While evaluating pending events if something failed (unlikely) or if we were
10370	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10371	*/
10372	if (rcStrict != VINF_SUCCESS)
10373	return rcStrict;
10374	if ( pVmxTransient->fIsNestedGuest
10375	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10376	{
10377	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10378	return VINF_VMX_VMEXIT;
10379	}
10380	#else
10381	Assert(rcStrict == VINF_SUCCESS);
10382	#endif
10383
10384	/*
10385	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10386	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10387	* also result in triple-faulting the VM.
10388	*
10389	* With nested-guests, the above does not apply since unrestricted guest execution is a
10390	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10391	*/
10392	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10393	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10394	{ /* likely */ }
10395	else
10396	{
10397	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10398	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10399	return rcStrict;
10400	}
10401
10402	/*
10403	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10404	* import CR3 themselves. We will need to update them here, as even as late as the above
10405	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10406	* the below force flags to be set.
10407	*/
10408	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10409	{
10410	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10411	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10412	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10413	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10414	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10415	}
10416	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10417	{
10418	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10419	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10420	}
10421
10422	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10423	/* Paranoia. */
10424	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10425	#endif
10426
10427	/*
10428	* No longjmps to ring-3 from this point on!!!
10429	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10430	* This also disables flushing of the R0-logger instance (if any).
10431	*/
10432	VMMRZCallRing3Disable(pVCpu);
10433
10434	/*
10435	* Export the guest state bits.
10436	*
10437	* We cannot perform longjmps while loading the guest state because we do not preserve the
10438	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10439	* CPU migration.
10440	*
10441	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10442	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10443	*/
10444	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10445	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10446	{ /* likely */ }
10447	else
10448	{
10449	VMMRZCallRing3Enable(pVCpu);
10450	return rcStrict;
10451	}
10452
10453	/*
10454	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10455	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10456	* preemption disabled for a while. Since this is purely to aid the
10457	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10458	* disable interrupt on NT.
10459	*
10460	* We need to check for force-flags that could've possible been altered since we last
10461	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10462	* see @bugref{6398}).
10463	*
10464	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10465	* to ring-3 before executing guest code.
10466	*/
10467	pVmxTransient->fEFlags = ASMIntDisableFlags();
10468
10469	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10470	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10471	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10472	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10473	{
10474	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10475	{
10476	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10477	/*
10478	* If we are executing a nested-guest make sure that we should intercept subsequent
10479	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10480	* the VM-exit instruction emulation happy.
10481	*/
10482	if (pVmxTransient->fIsNestedGuest)
10483	CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
10484	#endif
10485
10486	/*
10487	* We've injected any pending events. This is really the point of no return (to ring-3).
10488	*
10489	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10490	* returns from this function, so do -not- enable them here.
10491	*/
10492	pVCpu->hm.s.Event.fPending = false;
10493	return VINF_SUCCESS;
10494	}
10495
10496	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10497	rcStrict = VINF_EM_RAW_INTERRUPT;
10498	}
10499	else
10500	{
10501	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10502	rcStrict = VINF_EM_RAW_TO_R3;
10503	}
10504
10505	ASMSetFlags(pVmxTransient->fEFlags);
10506	VMMRZCallRing3Enable(pVCpu);
10507
10508	return rcStrict;
10509	}
10510
10511
10512	/**
10513	* Final preparations before executing guest code using hardware-assisted VMX.
10514	*
10515	* We can no longer get preempted to a different host CPU and there are no returns
10516	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10517	* failures), this function is not intended to fail sans unrecoverable hardware
10518	* errors.
10519	*
10520	* @param pVCpu The cross context virtual CPU structure.
10521	* @param pVmxTransient The VMX-transient structure.
10522	*
10523	* @remarks Called with preemption disabled.
10524	* @remarks No-long-jump zone!!!
10525	*/
10526	static void hmR0VmxPreRunGuestCommitted(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10527	{
10528	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10529	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10530	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10531	Assert(!pVCpu->hm.s.Event.fPending);
10532
10533	/*
10534	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10535	*/
10536	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10537	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10538
10539	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10540	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10541	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10542	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10543
10544	if (!CPUMIsGuestFPUStateActive(pVCpu))
10545	{
10546	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10547	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10548	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10549	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10550	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10551	}
10552
10553	/*
10554	* Re-export the host state bits as we may've been preempted (only happens when
10555	* thread-context hooks are used or when the VM start function changes) or if
10556	* the host CR0 is modified while loading the guest FPU state above.
10557	*
10558	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10559	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10560	* see @bugref{8432}.
10561	*
10562	* This may also happen when switching to/from a nested-guest VMCS without leaving
10563	* ring-0.
10564	*/
10565	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10566	{
10567	hmR0VmxExportHostState(pVCpu);
10568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10569	}
10570	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10571
10572	/*
10573	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10574	*/
10575	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10576	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10577	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10578
10579	/*
10580	* Store status of the shared guest/host debug state at the time of VM-entry.
10581	*/
10582	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10583	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10584
10585	/*
10586	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10587	* more than one conditional check. The post-run side of our code shall determine
10588	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10589	*/
10590	if (pVmcsInfo->pbVirtApic)
10591	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10592
10593	/*
10594	* Update the host MSRs values in the VM-exit MSR-load area.
10595	*/
10596	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10597	{
10598	if (pVmcsInfo->cExitMsrLoad > 0)
10599	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10600	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10601	}
10602
10603	/*
10604	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10605	* VMX-preemption timer based on the next virtual sync clock deadline.
10606	*/
10607	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10608	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10609	{
10610	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10611	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10612	}
10613
10614	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10615	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10616	if (!fIsRdtscIntercepted)
10617	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10618	else
10619	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10620
10621	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10622	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10623	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10624	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Record the error reporting info. with the current host CPU. */
10625	pVmcsInfo->idHostCpuState = idCurrentCpu; /* Record the CPU for which the host-state has been exported. */
10626	pVmcsInfo->idHostCpuExec = idCurrentCpu; /* Record the CPU on which we shall execute. */
10627
10628	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10629
10630	TMNotifyStartOfExecution(pVM, pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10631	as we're about to start executing the guest. */
10632
10633	/*
10634	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10635	*
10636	* This is done this late as updating the TSC offsetting/preemption timer above
10637	* figures out if we can skip intercepting RDTSCP by calculating the number of
10638	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10639	*/
10640	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10641	&& !fIsRdtscIntercepted)
10642	{
10643	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10644
10645	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10646	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10647	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10648	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10649	AssertRC(rc);
10650	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10651	pVmxTransient->fRemoveTscAuxMsr = true;
10652	}
10653
10654	#ifdef VBOX_STRICT
10655	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10656	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10657	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10658	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10659	#endif
10660
10661	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10662	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10663	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10664	* see @bugref{9180#c54}. */
10665	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10666	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10667	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10668	#endif
10669	}
10670
10671
10672	/**
10673	* First C routine invoked after running guest code using hardware-assisted VMX.
10674	*
10675	* @param pVCpu The cross context virtual CPU structure.
10676	* @param pVmxTransient The VMX-transient structure.
10677	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10678	*
10679	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10680	*
10681	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10682	* unconditionally when it is safe to do so.
10683	*/
10684	static void hmR0VmxPostRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10685	{
10686	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10687
10688	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10689	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10690	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10691	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10692	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10693	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10694
10695	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10696	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10697	{
10698	uint64_t uGstTsc;
10699	if (!pVmxTransient->fIsNestedGuest)
10700	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10701	else
10702	{
10703	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10704	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10705	}
10706	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10707	}
10708
10709	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10710	TMNotifyEndOfExecution(pVCpu->CTX_SUFF(pVM), pVCpu); /* Notify TM that the guest is no longer running. */
10711	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10712
10713	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10714	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10715	#ifdef VBOX_STRICT
10716	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10717	#endif
10718	Assert(!ASMIntAreEnabled());
10719	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10720	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10721
10722	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
10723	/*
10724	* Clean all the VMCS fields in the transient structure before reading
10725	* anything from the VMCS.
10726	*/
10727	pVmxTransient->uExitReason = 0;
10728	pVmxTransient->uExitIntErrorCode = 0;
10729	pVmxTransient->uExitQual = 0;
10730	pVmxTransient->uGuestLinearAddr = 0;
10731	pVmxTransient->uExitIntInfo = 0;
10732	pVmxTransient->cbExitInstr = 0;
10733	pVmxTransient->ExitInstrInfo.u = 0;
10734	pVmxTransient->uEntryIntInfo = 0;
10735	pVmxTransient->uEntryXcptErrorCode = 0;
10736	pVmxTransient->cbEntryInstr = 0;
10737	pVmxTransient->uIdtVectoringInfo = 0;
10738	pVmxTransient->uIdtVectoringErrorCode = 0;
10739	#endif
10740
10741	/*
10742	* Save the basic VM-exit reason and check if the VM-entry failed.
10743	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10744	*/
10745	uint32_t uExitReason;
10746	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10747	AssertRC(rc);
10748	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10749	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10750
10751	/*
10752	* Log the VM-exit before logging anything else as otherwise it might be a
10753	* tad confusing what happens before and after the world-switch.
10754	*/
10755	HMVMX_LOG_EXIT(pVCpu, uExitReason);
10756
10757	/*
10758	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
10759	* bitmap permissions, if it was added before VM-entry.
10760	*/
10761	if (pVmxTransient->fRemoveTscAuxMsr)
10762	{
10763	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10764	pVmxTransient->fRemoveTscAuxMsr = false;
10765	}
10766
10767	/*
10768	* Check if VMLAUNCH/VMRESUME succeeded.
10769	* If this failed, we cause a guru meditation and cease further execution.
10770	*
10771	* However, if we are executing a nested-guest we might fail if we use the
10772	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10773	*/
10774	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10775	{
10776	/*
10777	* Update the VM-exit history array here even if the VM-entry failed due to:
10778	* - Invalid guest state.
10779	* - MSR loading.
10780	* - Machine-check event.
10781	*
10782	* In any of the above cases we will still have a "valid" VM-exit reason
10783	* despite @a fVMEntryFailed being false.
10784	*
10785	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10786	*
10787	* Note! We don't have CS or RIP at this point. Will probably address that later
10788	* by amending the history entry added here.
10789	*/
10790	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10791	UINT64_MAX, uHostTsc);
10792
10793	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10794	{
10795	VMMRZCallRing3Enable(pVCpu);
10796
10797	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10798	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10799
10800	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
10801	hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
10802	#endif
10803	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10804	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10805	AssertRC(rc);
10806	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10807	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10808	AssertRC(rc);
10809	#else
10810	/*
10811	* Import the guest-interruptibility state always as we need it while evaluating
10812	* injecting events on re-entry.
10813	*
10814	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10815	* checking for real-mode while exporting the state because all bits that cause
10816	* mode changes wrt CR0 are intercepted.
10817	*/
10818	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10819	AssertRC(rc);
10820	#endif
10821
10822	/*
10823	* Sync the TPR shadow with our APIC state.
10824	*/
10825	if ( !pVmxTransient->fIsNestedGuest
10826	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10827	{
10828	Assert(pVmcsInfo->pbVirtApic);
10829	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10830	{
10831	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10832	AssertRC(rc);
10833	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10834	}
10835	}
10836
10837	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10838	return;
10839	}
10840	}
10841	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10842	else if (pVmxTransient->fIsNestedGuest)
10843	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10844	#endif
10845	else
10846	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10847
10848	VMMRZCallRing3Enable(pVCpu);
10849	}
10850
10851
10852	/**
10853	* Runs the guest code using hardware-assisted VMX the normal way.
10854	*
10855	* @returns VBox status code.
10856	* @param pVCpu The cross context virtual CPU structure.
10857	* @param pcLoops Pointer to the number of executed loops.
10858	*/
10859	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
10860	{
10861	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10862	Assert(pcLoops);
10863	Assert(*pcLoops <= cMaxResumeLoops);
10864	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10865
10866	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10867	/*
10868	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
10869	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
10870	* guest VMCS while entering the VMX ring-0 session.
10871	*/
10872	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10873	{
10874	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
10875	if (RT_SUCCESS(rc))
10876	{ /* likely */ }
10877	else
10878	{
10879	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
10880	return rc;
10881	}
10882	}
10883	#endif
10884
10885	VMXTRANSIENT VmxTransient;
10886	RT_ZERO(VmxTransient);
10887	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10888
10889	/* Paranoia. */
10890	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10891
10892	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10893	for (;;)
10894	{
10895	Assert(!HMR0SuspendPending());
10896	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10897	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10898
10899	/*
10900	* Preparatory work for running nested-guest code, this may force us to
10901	* return to ring-3.
10902	*
10903	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10904	*/
10905	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10906	if (rcStrict != VINF_SUCCESS)
10907	break;
10908
10909	/* Interrupts are disabled at this point! */
10910	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10911	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10912	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10913	/* Interrupts are re-enabled at this point! */
10914
10915	/*
10916	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10917	*/
10918	if (RT_SUCCESS(rcRun))
10919	{ /* very likely */ }
10920	else
10921	{
10922	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10923	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10924	return rcRun;
10925	}
10926
10927	/*
10928	* Profile the VM-exit.
10929	*/
10930	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10932	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10933	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10934	HMVMX_START_EXIT_DISPATCH_PROF();
10935
10936	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10937
10938	/*
10939	* Handle the VM-exit.
10940	*/
10941	#ifdef HMVMX_USE_FUNCTION_TABLE
10942	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10943	#else
10944	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10945	#endif
10946	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10947	if (rcStrict == VINF_SUCCESS)
10948	{
10949	if (++(*pcLoops) <= cMaxResumeLoops)
10950	continue;
10951	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10952	rcStrict = VINF_EM_RAW_INTERRUPT;
10953	}
10954	break;
10955	}
10956
10957	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10958	return rcStrict;
10959	}
10960
10961
10962	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10963	/**
10964	* Runs the nested-guest code using hardware-assisted VMX.
10965	*
10966	* @returns VBox status code.
10967	* @param pVCpu The cross context virtual CPU structure.
10968	* @param pcLoops Pointer to the number of executed loops.
10969	*
10970	* @sa hmR0VmxRunGuestCodeNormal.
10971	*/
10972	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
10973	{
10974	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10975	Assert(pcLoops);
10976	Assert(*pcLoops <= cMaxResumeLoops);
10977	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10978
10979	/*
10980	* Switch to the nested-guest VMCS as we may have transitioned from executing the
10981	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
10982	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
10983	*/
10984	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10985	{
10986	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
10987	if (RT_SUCCESS(rc))
10988	{ /* likely */ }
10989	else
10990	{
10991	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
10992	return rc;
10993	}
10994	}
10995
10996	VMXTRANSIENT VmxTransient;
10997	RT_ZERO(VmxTransient);
10998	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10999	VmxTransient.fIsNestedGuest = true;
11000
11001	/* Paranoia. */
11002	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11003
11004	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11005	for (;;)
11006	{
11007	Assert(!HMR0SuspendPending());
11008	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11009	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11010
11011	/*
11012	* Preparatory work for running guest code, this may force us to
11013	* return to ring-3.
11014	*
11015	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11016	*/
11017	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11018	if (rcStrict != VINF_SUCCESS)
11019	break;
11020
11021	/* Interrupts are disabled at this point! */
11022	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11023	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11024	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11025	/* Interrupts are re-enabled at this point! */
11026
11027	/*
11028	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11029	*/
11030	if (RT_SUCCESS(rcRun))
11031	{ /* very likely */ }
11032	else
11033	{
11034	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11035	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11036	return rcRun;
11037	}
11038
11039	/*
11040	* Profile the VM-exit.
11041	*/
11042	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11044	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11045	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11046	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11047	HMVMX_START_EXIT_DISPATCH_PROF();
11048
11049	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11050
11051	/*
11052	* Handle the VM-exit.
11053	*/
11054	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11055	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11056	if (rcStrict == VINF_SUCCESS)
11057	{
11058	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11059	{
11060	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11061	rcStrict = VINF_VMX_VMEXIT;
11062	}
11063	else
11064	{
11065	if (++(*pcLoops) <= cMaxResumeLoops)
11066	continue;
11067	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11068	rcStrict = VINF_EM_RAW_INTERRUPT;
11069	}
11070	}
11071	else
11072	Assert(rcStrict != VINF_VMX_VMEXIT);
11073	break;
11074	}
11075
11076	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11077	return rcStrict;
11078	}
11079	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11080
11081
11082	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11083	* probes.
11084	*
11085	* The following few functions and associated structure contains the bloat
11086	* necessary for providing detailed debug events and dtrace probes as well as
11087	* reliable host side single stepping. This works on the principle of
11088	* "subclassing" the normal execution loop and workers. We replace the loop
11089	* method completely and override selected helpers to add necessary adjustments
11090	* to their core operation.
11091	*
11092	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11093	* any performance for debug and analysis features.
11094	*
11095	* @{
11096	*/
11097
11098	/**
11099	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11100	* the debug run loop.
11101	*/
11102	typedef struct VMXRUNDBGSTATE
11103	{
11104	/** The RIP we started executing at. This is for detecting that we stepped. */
11105	uint64_t uRipStart;
11106	/** The CS we started executing with. */
11107	uint16_t uCsStart;
11108
11109	/** Whether we've actually modified the 1st execution control field. */
11110	bool fModifiedProcCtls : 1;
11111	/** Whether we've actually modified the 2nd execution control field. */
11112	bool fModifiedProcCtls2 : 1;
11113	/** Whether we've actually modified the exception bitmap. */
11114	bool fModifiedXcptBitmap : 1;
11115
11116	/** We desire the modified the CR0 mask to be cleared. */
11117	bool fClearCr0Mask : 1;
11118	/** We desire the modified the CR4 mask to be cleared. */
11119	bool fClearCr4Mask : 1;
11120	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11121	uint32_t fCpe1Extra;
11122	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11123	uint32_t fCpe1Unwanted;
11124	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11125	uint32_t fCpe2Extra;
11126	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11127	uint32_t bmXcptExtra;
11128	/** The sequence number of the Dtrace provider settings the state was
11129	* configured against. */
11130	uint32_t uDtraceSettingsSeqNo;
11131	/** VM-exits to check (one bit per VM-exit). */
11132	uint32_t bmExitsToCheck[3];
11133
11134	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11135	uint32_t fProcCtlsInitial;
11136	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11137	uint32_t fProcCtls2Initial;
11138	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11139	uint32_t bmXcptInitial;
11140	} VMXRUNDBGSTATE;
11141	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11142	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11143
11144
11145	/**
11146	* Initializes the VMXRUNDBGSTATE structure.
11147	*
11148	* @param pVCpu The cross context virtual CPU structure of the
11149	* calling EMT.
11150	* @param pVmxTransient The VMX-transient structure.
11151	* @param pDbgState The debug state to initialize.
11152	*/
11153	static void hmR0VmxRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11154	{
11155	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11156	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11157
11158	pDbgState->fModifiedProcCtls = false;
11159	pDbgState->fModifiedProcCtls2 = false;
11160	pDbgState->fModifiedXcptBitmap = false;
11161	pDbgState->fClearCr0Mask = false;
11162	pDbgState->fClearCr4Mask = false;
11163	pDbgState->fCpe1Extra = 0;
11164	pDbgState->fCpe1Unwanted = 0;
11165	pDbgState->fCpe2Extra = 0;
11166	pDbgState->bmXcptExtra = 0;
11167	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11168	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11169	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11170	}
11171
11172
11173	/**
11174	* Updates the VMSC fields with changes requested by @a pDbgState.
11175	*
11176	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11177	* immediately before executing guest code, i.e. when interrupts are disabled.
11178	* We don't check status codes here as we cannot easily assert or return in the
11179	* latter case.
11180	*
11181	* @param pVCpu The cross context virtual CPU structure.
11182	* @param pVmxTransient The VMX-transient structure.
11183	* @param pDbgState The debug state.
11184	*/
11185	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11186	{
11187	/*
11188	* Ensure desired flags in VMCS control fields are set.
11189	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11190	*
11191	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11192	* there should be no stale data in pCtx at this point.
11193	*/
11194	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11195	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11196	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11197	{
11198	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11199	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11200	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11201	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11202	pDbgState->fModifiedProcCtls = true;
11203	}
11204
11205	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11206	{
11207	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11208	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11209	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11210	pDbgState->fModifiedProcCtls2 = true;
11211	}
11212
11213	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11214	{
11215	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11216	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11217	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11218	pDbgState->fModifiedXcptBitmap = true;
11219	}
11220
11221	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11222	{
11223	pVmcsInfo->u64Cr0Mask = 0;
11224	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, 0);
11225	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11226	}
11227
11228	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11229	{
11230	pVmcsInfo->u64Cr4Mask = 0;
11231	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, 0);
11232	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11233	}
11234
11235	NOREF(pVCpu);
11236	}
11237
11238
11239	/**
11240	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11241	* re-entry next time around.
11242	*
11243	* @returns Strict VBox status code (i.e. informational status codes too).
11244	* @param pVCpu The cross context virtual CPU structure.
11245	* @param pVmxTransient The VMX-transient structure.
11246	* @param pDbgState The debug state.
11247	* @param rcStrict The return code from executing the guest using single
11248	* stepping.
11249	*/
11250	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11251	VBOXSTRICTRC rcStrict)
11252	{
11253	/*
11254	* Restore VM-exit control settings as we may not reenter this function the
11255	* next time around.
11256	*/
11257	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11258
11259	/* We reload the initial value, trigger what we can of recalculations the
11260	next time around. From the looks of things, that's all that's required atm. */
11261	if (pDbgState->fModifiedProcCtls)
11262	{
11263	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11264	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11265	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11266	AssertRC(rc2);
11267	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11268	}
11269
11270	/* We're currently the only ones messing with this one, so just restore the
11271	cached value and reload the field. */
11272	if ( pDbgState->fModifiedProcCtls2
11273	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11274	{
11275	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11276	AssertRC(rc2);
11277	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11278	}
11279
11280	/* If we've modified the exception bitmap, we restore it and trigger
11281	reloading and partial recalculation the next time around. */
11282	if (pDbgState->fModifiedXcptBitmap)
11283	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11284
11285	return rcStrict;
11286	}
11287
11288
11289	/**
11290	* Configures VM-exit controls for current DBGF and DTrace settings.
11291	*
11292	* This updates @a pDbgState and the VMCS execution control fields to reflect
11293	* the necessary VM-exits demanded by DBGF and DTrace.
11294	*
11295	* @param pVCpu The cross context virtual CPU structure.
11296	* @param pVmxTransient The VMX-transient structure. May update
11297	* fUpdatedTscOffsettingAndPreemptTimer.
11298	* @param pDbgState The debug state.
11299	*/
11300	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11301	{
11302	/*
11303	* Take down the dtrace serial number so we can spot changes.
11304	*/
11305	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11306	ASMCompilerBarrier();
11307
11308	/*
11309	* We'll rebuild most of the middle block of data members (holding the
11310	* current settings) as we go along here, so start by clearing it all.
11311	*/
11312	pDbgState->bmXcptExtra = 0;
11313	pDbgState->fCpe1Extra = 0;
11314	pDbgState->fCpe1Unwanted = 0;
11315	pDbgState->fCpe2Extra = 0;
11316	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11317	pDbgState->bmExitsToCheck[i] = 0;
11318
11319	/*
11320	* Software interrupts (INT XXh) - no idea how to trigger these...
11321	*/
11322	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11323	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11324	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11325	{
11326	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11327	}
11328
11329	/*
11330	* INT3 breakpoints - triggered by #BP exceptions.
11331	*/
11332	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11333	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11334
11335	/*
11336	* Exception bitmap and XCPT events+probes.
11337	*/
11338	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11339	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11340	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11341
11342	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11343	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11344	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11345	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11346	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11347	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11348	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11349	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11350	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11351	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11352	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11353	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11354	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11355	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11356	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11357	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11358	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11359	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11360
11361	if (pDbgState->bmXcptExtra)
11362	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11363
11364	/*
11365	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11366	*
11367	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11368	* So, when adding/changing/removing please don't forget to update it.
11369	*
11370	* Some of the macros are picking up local variables to save horizontal space,
11371	* (being able to see it in a table is the lesser evil here).
11372	*/
11373	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11374	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11375	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11376	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11377	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11378	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11379	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11380	} else do { } while (0)
11381	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11382	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11383	{ \
11384	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11385	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11386	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11387	} else do { } while (0)
11388	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11389	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11390	{ \
11391	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11392	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11393	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11394	} else do { } while (0)
11395	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11396	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11397	{ \
11398	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11399	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11400	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11401	} else do { } while (0)
11402
11403	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11404	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11405	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11406	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11407	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11408
11409	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11410	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11411	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11412	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11413	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11414	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11415	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11416	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11417	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11418	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11419	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11420	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11421	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11422	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11423	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11424	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11425	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11426	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11427	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11428	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11429	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11430	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11431	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11432	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11433	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11434	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11435	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11436	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11437	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11438	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11439	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11440	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11441	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11442	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11443	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11444	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11445
11446	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11447	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11448	{
11449	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11450	\| CPUMCTX_EXTRN_APIC_TPR);
11451	AssertRC(rc);
11452
11453	#if 0 /** @todo fix me */
11454	pDbgState->fClearCr0Mask = true;
11455	pDbgState->fClearCr4Mask = true;
11456	#endif
11457	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11458	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11459	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11460	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11461	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11462	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11463	require clearing here and in the loop if we start using it. */
11464	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11465	}
11466	else
11467	{
11468	if (pDbgState->fClearCr0Mask)
11469	{
11470	pDbgState->fClearCr0Mask = false;
11471	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11472	}
11473	if (pDbgState->fClearCr4Mask)
11474	{
11475	pDbgState->fClearCr4Mask = false;
11476	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11477	}
11478	}
11479	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11480	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11481
11482	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11483	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11484	{
11485	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11486	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11487	}
11488	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11489	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11490
11491	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11492	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11493	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11494	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11495	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11496	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11497	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11498	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11499	#if 0 /** @todo too slow, fix handler. */
11500	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11501	#endif
11502	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11503
11504	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11505	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11506	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11507	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11508	{
11509	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11510	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11511	}
11512	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11513	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11514	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11515	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11516
11517	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11518	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11519	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11520	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11521	{
11522	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11523	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11524	}
11525	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11526	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11527	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11528	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11529
11530	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11531	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11532	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11533	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11534	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11535	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11536	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11537	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11538	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11539	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11540	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11541	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11542	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11543	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11544	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11545	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11546	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11547	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11548	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11549	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11550	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11551	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11552
11553	#undef IS_EITHER_ENABLED
11554	#undef SET_ONLY_XBM_IF_EITHER_EN
11555	#undef SET_CPE1_XBM_IF_EITHER_EN
11556	#undef SET_CPEU_XBM_IF_EITHER_EN
11557	#undef SET_CPE2_XBM_IF_EITHER_EN
11558
11559	/*
11560	* Sanitize the control stuff.
11561	*/
11562	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11563	if (pDbgState->fCpe2Extra)
11564	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11565	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11566	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11567	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11568	{
11569	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11570	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11571	}
11572
11573	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11574	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11575	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11576	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11577	}
11578
11579
11580	/**
11581	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11582	* appropriate.
11583	*
11584	* The caller has checked the VM-exit against the
11585	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11586	* already, so we don't have to do that either.
11587	*
11588	* @returns Strict VBox status code (i.e. informational status codes too).
11589	* @param pVCpu The cross context virtual CPU structure.
11590	* @param pVmxTransient The VMX-transient structure.
11591	* @param uExitReason The VM-exit reason.
11592	*
11593	* @remarks The name of this function is displayed by dtrace, so keep it short
11594	* and to the point. No longer than 33 chars long, please.
11595	*/
11596	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11597	{
11598	/*
11599	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11600	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11601	*
11602	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11603	* does. Must add/change/remove both places. Same ordering, please.
11604	*
11605	* Added/removed events must also be reflected in the next section
11606	* where we dispatch dtrace events.
11607	*/
11608	bool fDtrace1 = false;
11609	bool fDtrace2 = false;
11610	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11611	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11612	uint32_t uEventArg = 0;
11613	#define SET_EXIT(a_EventSubName) \
11614	do { \
11615	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11616	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11617	} while (0)
11618	#define SET_BOTH(a_EventSubName) \
11619	do { \
11620	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11621	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11622	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11623	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11624	} while (0)
11625	switch (uExitReason)
11626	{
11627	case VMX_EXIT_MTF:
11628	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11629
11630	case VMX_EXIT_XCPT_OR_NMI:
11631	{
11632	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11633	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11634	{
11635	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11636	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11637	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11638	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11639	{
11640	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11641	{
11642	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11643	uEventArg = pVmxTransient->uExitIntErrorCode;
11644	}
11645	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11646	switch (enmEvent1)
11647	{
11648	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11649	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11650	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11651	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11652	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11653	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11654	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11655	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11656	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11657	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11658	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11659	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11660	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11661	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11662	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11663	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11664	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11665	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11666	default: break;
11667	}
11668	}
11669	else
11670	AssertFailed();
11671	break;
11672
11673	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11674	uEventArg = idxVector;
11675	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11676	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11677	break;
11678	}
11679	break;
11680	}
11681
11682	case VMX_EXIT_TRIPLE_FAULT:
11683	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11684	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11685	break;
11686	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11687	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11688	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11689	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11690	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11691
11692	/* Instruction specific VM-exits: */
11693	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11694	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11695	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11696	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11697	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11698	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11699	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11700	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11701	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11702	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11703	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11704	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11705	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11706	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11707	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11708	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11709	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11710	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11711	case VMX_EXIT_MOV_CRX:
11712	hmR0VmxReadExitQualVmcs(pVmxTransient);
11713	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11714	SET_BOTH(CRX_READ);
11715	else
11716	SET_BOTH(CRX_WRITE);
11717	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11718	break;
11719	case VMX_EXIT_MOV_DRX:
11720	hmR0VmxReadExitQualVmcs(pVmxTransient);
11721	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11722	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11723	SET_BOTH(DRX_READ);
11724	else
11725	SET_BOTH(DRX_WRITE);
11726	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11727	break;
11728	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11729	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11730	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11731	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11732	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11733	case VMX_EXIT_GDTR_IDTR_ACCESS:
11734	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11735	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11736	{
11737	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11738	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11739	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11740	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11741	}
11742	break;
11743
11744	case VMX_EXIT_LDTR_TR_ACCESS:
11745	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11746	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11747	{
11748	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11749	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11750	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11751	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11752	}
11753	break;
11754
11755	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11756	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11757	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11758	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11759	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11760	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11761	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11762	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11763	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11764	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11765	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11766
11767	/* Events that aren't relevant at this point. */
11768	case VMX_EXIT_EXT_INT:
11769	case VMX_EXIT_INT_WINDOW:
11770	case VMX_EXIT_NMI_WINDOW:
11771	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11772	case VMX_EXIT_PREEMPT_TIMER:
11773	case VMX_EXIT_IO_INSTR:
11774	break;
11775
11776	/* Errors and unexpected events. */
11777	case VMX_EXIT_INIT_SIGNAL:
11778	case VMX_EXIT_SIPI:
11779	case VMX_EXIT_IO_SMI:
11780	case VMX_EXIT_SMI:
11781	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11782	case VMX_EXIT_ERR_MSR_LOAD:
11783	case VMX_EXIT_ERR_MACHINE_CHECK:
11784	case VMX_EXIT_PML_FULL:
11785	case VMX_EXIT_VIRTUALIZED_EOI:
11786	break;
11787
11788	default:
11789	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11790	break;
11791	}
11792	#undef SET_BOTH
11793	#undef SET_EXIT
11794
11795	/*
11796	* Dtrace tracepoints go first. We do them here at once so we don't
11797	* have to copy the guest state saving and stuff a few dozen times.
11798	* Down side is that we've got to repeat the switch, though this time
11799	* we use enmEvent since the probes are a subset of what DBGF does.
11800	*/
11801	if (fDtrace1 \|\| fDtrace2)
11802	{
11803	hmR0VmxReadExitQualVmcs(pVmxTransient);
11804	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11805	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11806	switch (enmEvent1)
11807	{
11808	/** @todo consider which extra parameters would be helpful for each probe. */
11809	case DBGFEVENT_END: break;
11810	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11811	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11812	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11813	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11814	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11815	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11816	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11817	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11818	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11819	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11820	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11821	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11822	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11823	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11824	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11825	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11826	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11827	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11828	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11829	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11830	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11831	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11832	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11833	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11834	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11835	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11836	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11837	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11838	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11839	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11840	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11841	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11842	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11843	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11844	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11845	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11846	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11847	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11848	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11849	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11850	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11851	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11852	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11853	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11854	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11855	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11856	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11857	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11858	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11859	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11860	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11861	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11862	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11863	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11864	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11865	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11866	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11867	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11868	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11869	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11870	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11871	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11872	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11873	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11874	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11875	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11876	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11877	}
11878	switch (enmEvent2)
11879	{
11880	/** @todo consider which extra parameters would be helpful for each probe. */
11881	case DBGFEVENT_END: break;
11882	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11883	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11884	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11885	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11886	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11887	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11888	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11889	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11890	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11891	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11892	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11893	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11894	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11895	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11896	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11897	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11898	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11899	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11900	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11901	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11902	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11903	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11904	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11905	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11906	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11907	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11908	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11909	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11910	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11911	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11912	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11913	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11914	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11915	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11916	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11917	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11918	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11919	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11920	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11921	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11922	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11923	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11924	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11925	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11926	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11927	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11928	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11929	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11930	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11931	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11932	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11933	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11934	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11935	}
11936	}
11937
11938	/*
11939	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11940	* the DBGF call will do a full check).
11941	*
11942	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11943	* Note! If we have to events, we prioritize the first, i.e. the instruction
11944	* one, in order to avoid event nesting.
11945	*/
11946	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11947	if ( enmEvent1 != DBGFEVENT_END
11948	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11949	{
11950	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11951	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11952	if (rcStrict != VINF_SUCCESS)
11953	return rcStrict;
11954	}
11955	else if ( enmEvent2 != DBGFEVENT_END
11956	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11957	{
11958	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11959	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11960	if (rcStrict != VINF_SUCCESS)
11961	return rcStrict;
11962	}
11963
11964	return VINF_SUCCESS;
11965	}
11966
11967
11968	/**
11969	* Single-stepping VM-exit filtering.
11970	*
11971	* This is preprocessing the VM-exits and deciding whether we've gotten far
11972	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11973	* handling is performed.
11974	*
11975	* @returns Strict VBox status code (i.e. informational status codes too).
11976	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11977	* @param pVmxTransient The VMX-transient structure.
11978	* @param pDbgState The debug state.
11979	*/
11980	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11981	{
11982	/*
11983	* Expensive (saves context) generic dtrace VM-exit probe.
11984	*/
11985	uint32_t const uExitReason = pVmxTransient->uExitReason;
11986	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11987	{ /* more likely */ }
11988	else
11989	{
11990	hmR0VmxReadExitQualVmcs(pVmxTransient);
11991	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11992	AssertRC(rc);
11993	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11994	}
11995
11996	/*
11997	* Check for host NMI, just to get that out of the way.
11998	*/
11999	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12000	{ /* normally likely */ }
12001	else
12002	{
12003	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12004	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12005	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12006	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12007	}
12008
12009	/*
12010	* Check for single stepping event if we're stepping.
12011	*/
12012	if (pVCpu->hm.s.fSingleInstruction)
12013	{
12014	switch (uExitReason)
12015	{
12016	case VMX_EXIT_MTF:
12017	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12018
12019	/* Various events: */
12020	case VMX_EXIT_XCPT_OR_NMI:
12021	case VMX_EXIT_EXT_INT:
12022	case VMX_EXIT_TRIPLE_FAULT:
12023	case VMX_EXIT_INT_WINDOW:
12024	case VMX_EXIT_NMI_WINDOW:
12025	case VMX_EXIT_TASK_SWITCH:
12026	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12027	case VMX_EXIT_APIC_ACCESS:
12028	case VMX_EXIT_EPT_VIOLATION:
12029	case VMX_EXIT_EPT_MISCONFIG:
12030	case VMX_EXIT_PREEMPT_TIMER:
12031
12032	/* Instruction specific VM-exits: */
12033	case VMX_EXIT_CPUID:
12034	case VMX_EXIT_GETSEC:
12035	case VMX_EXIT_HLT:
12036	case VMX_EXIT_INVD:
12037	case VMX_EXIT_INVLPG:
12038	case VMX_EXIT_RDPMC:
12039	case VMX_EXIT_RDTSC:
12040	case VMX_EXIT_RSM:
12041	case VMX_EXIT_VMCALL:
12042	case VMX_EXIT_VMCLEAR:
12043	case VMX_EXIT_VMLAUNCH:
12044	case VMX_EXIT_VMPTRLD:
12045	case VMX_EXIT_VMPTRST:
12046	case VMX_EXIT_VMREAD:
12047	case VMX_EXIT_VMRESUME:
12048	case VMX_EXIT_VMWRITE:
12049	case VMX_EXIT_VMXOFF:
12050	case VMX_EXIT_VMXON:
12051	case VMX_EXIT_MOV_CRX:
12052	case VMX_EXIT_MOV_DRX:
12053	case VMX_EXIT_IO_INSTR:
12054	case VMX_EXIT_RDMSR:
12055	case VMX_EXIT_WRMSR:
12056	case VMX_EXIT_MWAIT:
12057	case VMX_EXIT_MONITOR:
12058	case VMX_EXIT_PAUSE:
12059	case VMX_EXIT_GDTR_IDTR_ACCESS:
12060	case VMX_EXIT_LDTR_TR_ACCESS:
12061	case VMX_EXIT_INVEPT:
12062	case VMX_EXIT_RDTSCP:
12063	case VMX_EXIT_INVVPID:
12064	case VMX_EXIT_WBINVD:
12065	case VMX_EXIT_XSETBV:
12066	case VMX_EXIT_RDRAND:
12067	case VMX_EXIT_INVPCID:
12068	case VMX_EXIT_VMFUNC:
12069	case VMX_EXIT_RDSEED:
12070	case VMX_EXIT_XSAVES:
12071	case VMX_EXIT_XRSTORS:
12072	{
12073	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12074	AssertRCReturn(rc, rc);
12075	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12076	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12077	return VINF_EM_DBG_STEPPED;
12078	break;
12079	}
12080
12081	/* Errors and unexpected events: */
12082	case VMX_EXIT_INIT_SIGNAL:
12083	case VMX_EXIT_SIPI:
12084	case VMX_EXIT_IO_SMI:
12085	case VMX_EXIT_SMI:
12086	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12087	case VMX_EXIT_ERR_MSR_LOAD:
12088	case VMX_EXIT_ERR_MACHINE_CHECK:
12089	case VMX_EXIT_PML_FULL:
12090	case VMX_EXIT_VIRTUALIZED_EOI:
12091	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12092	break;
12093
12094	default:
12095	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12096	break;
12097	}
12098	}
12099
12100	/*
12101	* Check for debugger event breakpoints and dtrace probes.
12102	*/
12103	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12104	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12105	{
12106	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12107	if (rcStrict != VINF_SUCCESS)
12108	return rcStrict;
12109	}
12110
12111	/*
12112	* Normal processing.
12113	*/
12114	#ifdef HMVMX_USE_FUNCTION_TABLE
12115	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12116	#else
12117	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12118	#endif
12119	}
12120
12121
12122	/**
12123	* Single steps guest code using hardware-assisted VMX.
12124	*
12125	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12126	* but single-stepping through the hypervisor debugger.
12127	*
12128	* @returns Strict VBox status code (i.e. informational status codes too).
12129	* @param pVCpu The cross context virtual CPU structure.
12130	* @param pcLoops Pointer to the number of executed loops.
12131	*
12132	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12133	*/
12134	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
12135	{
12136	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12137	Assert(pcLoops);
12138	Assert(*pcLoops <= cMaxResumeLoops);
12139
12140	VMXTRANSIENT VmxTransient;
12141	RT_ZERO(VmxTransient);
12142	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12143
12144	/* Set HMCPU indicators. */
12145	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12146	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12147	pVCpu->hm.s.fDebugWantRdTscExit = false;
12148	pVCpu->hm.s.fUsingDebugLoop = true;
12149
12150	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12151	VMXRUNDBGSTATE DbgState;
12152	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12153	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12154
12155	/*
12156	* The loop.
12157	*/
12158	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12159	for (;;)
12160	{
12161	Assert(!HMR0SuspendPending());
12162	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12163	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12164	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12165
12166	/* Set up VM-execution controls the next two can respond to. */
12167	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12168
12169	/*
12170	* Preparatory work for running guest code, this may force us to
12171	* return to ring-3.
12172	*
12173	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12174	*/
12175	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12176	if (rcStrict != VINF_SUCCESS)
12177	break;
12178
12179	/* Interrupts are disabled at this point! */
12180	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12181
12182	/* Override any obnoxious code in the above two calls. */
12183	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12184
12185	/*
12186	* Finally execute the guest.
12187	*/
12188	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12189
12190	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12191	/* Interrupts are re-enabled at this point! */
12192
12193	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12194	if (RT_SUCCESS(rcRun))
12195	{ /* very likely */ }
12196	else
12197	{
12198	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12199	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12200	return rcRun;
12201	}
12202
12203	/* Profile the VM-exit. */
12204	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12205	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12206	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12207	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12208	HMVMX_START_EXIT_DISPATCH_PROF();
12209
12210	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12211
12212	/*
12213	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12214	*/
12215	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12216	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12217	if (rcStrict != VINF_SUCCESS)
12218	break;
12219	if (++(*pcLoops) > cMaxResumeLoops)
12220	{
12221	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12222	rcStrict = VINF_EM_RAW_INTERRUPT;
12223	break;
12224	}
12225
12226	/*
12227	* Stepping: Did the RIP change, if so, consider it a single step.
12228	* Otherwise, make sure one of the TFs gets set.
12229	*/
12230	if (fStepping)
12231	{
12232	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12233	AssertRC(rc);
12234	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12235	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12236	{
12237	rcStrict = VINF_EM_DBG_STEPPED;
12238	break;
12239	}
12240	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12241	}
12242
12243	/*
12244	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12245	*/
12246	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12247	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12248	}
12249
12250	/*
12251	* Clear the X86_EFL_TF if necessary.
12252	*/
12253	if (pVCpu->hm.s.fClearTrapFlag)
12254	{
12255	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12256	AssertRC(rc);
12257	pVCpu->hm.s.fClearTrapFlag = false;
12258	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12259	}
12260	/** @todo there seems to be issues with the resume flag when the monitor trap
12261	* flag is pending without being used. Seen early in bios init when
12262	* accessing APIC page in protected mode. */
12263
12264	/*
12265	* Restore VM-exit control settings as we may not re-enter this function the
12266	* next time around.
12267	*/
12268	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12269
12270	/* Restore HMCPU indicators. */
12271	pVCpu->hm.s.fUsingDebugLoop = false;
12272	pVCpu->hm.s.fDebugWantRdTscExit = false;
12273	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12274
12275	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12276	return rcStrict;
12277	}
12278
12279
12280	/** @} */
12281
12282
12283	/**
12284	* Checks if any expensive dtrace probes are enabled and we should go to the
12285	* debug loop.
12286	*
12287	* @returns true if we should use debug loop, false if not.
12288	*/
12289	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12290	{
12291	/* It's probably faster to OR the raw 32-bit counter variables together.
12292	Since the variables are in an array and the probes are next to one
12293	another (more or less), we have good locality. So, better read
12294	eight-nine cache lines ever time and only have one conditional, than
12295	128+ conditionals, right? */
12296	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12297	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12298	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12299	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12300	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12301	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12302	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12303	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12304	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12305	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12306	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12307	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12308	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12309	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12310	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12311	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12312	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12313	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12314	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12315	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12316	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12317	) != 0
12318	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12319	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12320	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12321	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12322	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12323	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12324	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12325	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12326	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12327	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12328	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12329	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12330	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12331	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12332	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12333	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12334	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12335	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12336	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12337	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12338	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12339	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12340	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12341	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12342	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12343	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12344	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12345	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12346	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12347	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12348	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12349	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12350	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12351	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12352	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12353	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12354	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12355	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12356	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12357	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12358	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12359	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12360	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12361	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12362	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12363	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12364	) != 0
12365	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12366	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12367	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12368	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12369	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12370	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12371	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12372	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12373	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12374	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12375	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12376	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12377	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12378	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12379	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12380	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12381	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12382	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12383	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12384	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12385	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12386	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12387	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12388	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12389	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12390	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12391	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12392	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12393	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12394	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12395	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12396	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12397	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12398	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12399	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12400	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12401	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12402	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12403	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12404	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12405	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12406	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12407	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12408	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12409	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12410	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12411	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12412	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12413	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12414	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12415	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12416	) != 0;
12417	}
12418
12419
12420	/**
12421	* Runs the guest using hardware-assisted VMX.
12422	*
12423	* @returns Strict VBox status code (i.e. informational status codes too).
12424	* @param pVCpu The cross context virtual CPU structure.
12425	*/
12426	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu)
12427	{
12428	AssertPtr(pVCpu);
12429	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12430	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12431	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12432	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12433
12434	VBOXSTRICTRC rcStrict;
12435	uint32_t cLoops = 0;
12436	for (;;)
12437	{
12438	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12439	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12440	#else
12441	NOREF(pCtx);
12442	bool const fInNestedGuestMode = false;
12443	#endif
12444	if (!fInNestedGuestMode)
12445	{
12446	if ( !pVCpu->hm.s.fUseDebugLoop
12447	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12448	&& !DBGFIsStepping(pVCpu)
12449	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12450	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12451	else
12452	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12453	}
12454	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12455	else
12456	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12457
12458	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12459	{
12460	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12461	continue;
12462	}
12463	if (rcStrict == VINF_VMX_VMEXIT)
12464	{
12465	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12466	continue;
12467	}
12468	#endif
12469	break;
12470	}
12471
12472	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12473	switch (rcLoop)
12474	{
12475	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12476	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12477	}
12478
12479	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12480	if (RT_FAILURE(rc2))
12481	{
12482	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12483	rcStrict = rc2;
12484	}
12485	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12486	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12487	return rcStrict;
12488	}
12489
12490
12491	#ifndef HMVMX_USE_FUNCTION_TABLE
12492	/**
12493	* Handles a guest VM-exit from hardware-assisted VMX execution.
12494	*
12495	* @returns Strict VBox status code (i.e. informational status codes too).
12496	* @param pVCpu The cross context virtual CPU structure.
12497	* @param pVmxTransient The VMX-transient structure.
12498	*/
12499	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12500	{
12501	#ifdef DEBUG_ramshankar
12502	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12503	do { \
12504	if (a_fSave != 0) \
12505	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12506	VBOXSTRICTRC rcStrict = a_CallExpr; \
12507	if (a_fSave != 0) \
12508	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12509	return rcStrict; \
12510	} while (0)
12511	#else
12512	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12513	#endif
12514	uint32_t const uExitReason = pVmxTransient->uExitReason;
12515	switch (uExitReason)
12516	{
12517	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12518	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12519	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12520	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12521	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12522	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12523	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12524	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12525	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12526	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12527	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12528	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12529	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12530	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12531	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12532	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12533	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12534	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12535	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12536	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12537	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12538	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12539	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12540	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12541	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12542	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12543	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12544	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12545	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12546	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12547	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12548	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12549	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12550	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12551	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12552	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12553	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12554	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12555	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12556	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12557	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12558	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12559	#else
12560	case VMX_EXIT_VMCLEAR:
12561	case VMX_EXIT_VMLAUNCH:
12562	case VMX_EXIT_VMPTRLD:
12563	case VMX_EXIT_VMPTRST:
12564	case VMX_EXIT_VMREAD:
12565	case VMX_EXIT_VMRESUME:
12566	case VMX_EXIT_VMWRITE:
12567	case VMX_EXIT_VMXOFF:
12568	case VMX_EXIT_VMXON:
12569	case VMX_EXIT_INVVPID:
12570	case VMX_EXIT_INVEPT:
12571	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12572	#endif
12573
12574	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12575	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12576	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12577
12578	case VMX_EXIT_INIT_SIGNAL:
12579	case VMX_EXIT_SIPI:
12580	case VMX_EXIT_IO_SMI:
12581	case VMX_EXIT_SMI:
12582	case VMX_EXIT_ERR_MSR_LOAD:
12583	case VMX_EXIT_ERR_MACHINE_CHECK:
12584	case VMX_EXIT_PML_FULL:
12585	case VMX_EXIT_VIRTUALIZED_EOI:
12586	case VMX_EXIT_GDTR_IDTR_ACCESS:
12587	case VMX_EXIT_LDTR_TR_ACCESS:
12588	case VMX_EXIT_APIC_WRITE:
12589	case VMX_EXIT_RDRAND:
12590	case VMX_EXIT_RSM:
12591	case VMX_EXIT_VMFUNC:
12592	case VMX_EXIT_ENCLS:
12593	case VMX_EXIT_RDSEED:
12594	case VMX_EXIT_XSAVES:
12595	case VMX_EXIT_XRSTORS:
12596	case VMX_EXIT_UMWAIT:
12597	case VMX_EXIT_TPAUSE:
12598	default:
12599	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12600	}
12601	#undef VMEXIT_CALL_RET
12602	}
12603	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12604
12605
12606	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12607	/**
12608	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12609	*
12610	* @returns Strict VBox status code (i.e. informational status codes too).
12611	* @param pVCpu The cross context virtual CPU structure.
12612	* @param pVmxTransient The VMX-transient structure.
12613	*/
12614	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12615	{
12616	/** @todo NSTVMX: Remove after debugging page-fault issue. */
12617	#ifdef DEBUG_ramshankar
12618	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12619	Log4Func(("cs:rip=%#04x:%#RX64 rsp=%#RX64 eflags=%#RX32 cr3=%#RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12620	pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.cr3));
12621	#endif
12622
12623	uint32_t const uExitReason = pVmxTransient->uExitReason;
12624	switch (uExitReason)
12625	{
12626	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12627	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12628	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12629	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12630	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12631
12632	/*
12633	* We shouldn't direct host physical interrupts to the nested-guest.
12634	*/
12635	case VMX_EXIT_EXT_INT:
12636	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12637
12638	/*
12639	* Instructions that cause VM-exits unconditionally or the condition is
12640	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12641	* happens, it's guaranteed to be a nested-guest VM-exit).
12642	*
12643	* - Provides VM-exit instruction length ONLY.
12644	*/
12645	case VMX_EXIT_CPUID: /* Unconditional. */
12646	case VMX_EXIT_VMCALL:
12647	case VMX_EXIT_GETSEC:
12648	case VMX_EXIT_INVD:
12649	case VMX_EXIT_XSETBV:
12650	case VMX_EXIT_VMLAUNCH:
12651	case VMX_EXIT_VMRESUME:
12652	case VMX_EXIT_VMXOFF:
12653	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
12654	case VMX_EXIT_VMFUNC:
12655	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12656
12657	/*
12658	* Instructions that cause VM-exits unconditionally or the condition is
12659	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12660	* happens, it's guaranteed to be a nested-guest VM-exit).
12661	*
12662	* - Provides VM-exit instruction length.
12663	* - Provides VM-exit information.
12664	* - Optionally provides Exit qualification.
12665	*
12666	* Since Exit qualification is 0 for all VM-exits where it is not
12667	* applicable, reading and passing it to the guest should produce
12668	* defined behavior.
12669	*
12670	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12671	*/
12672	case VMX_EXIT_INVEPT: /* Unconditional. */
12673	case VMX_EXIT_INVVPID:
12674	case VMX_EXIT_VMCLEAR:
12675	case VMX_EXIT_VMPTRLD:
12676	case VMX_EXIT_VMPTRST:
12677	case VMX_EXIT_VMXON:
12678	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
12679	case VMX_EXIT_LDTR_TR_ACCESS:
12680	case VMX_EXIT_RDRAND:
12681	case VMX_EXIT_RDSEED:
12682	case VMX_EXIT_XSAVES:
12683	case VMX_EXIT_XRSTORS:
12684	case VMX_EXIT_UMWAIT:
12685	case VMX_EXIT_TPAUSE:
12686	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12687
12688	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12689	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12690	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12691	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12692	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12693	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12694	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12695	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12696	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12697	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12698	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12699	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12700	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12701	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12702	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12703	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12704	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12705	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12706	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12707
12708	case VMX_EXIT_PREEMPT_TIMER:
12709	{
12710	/** @todo NSTVMX: Preempt timer. */
12711	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12712	}
12713
12714	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12715	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12716
12717	case VMX_EXIT_VMREAD:
12718	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12719
12720	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12721	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12722
12723	case VMX_EXIT_INIT_SIGNAL:
12724	case VMX_EXIT_SIPI:
12725	case VMX_EXIT_IO_SMI:
12726	case VMX_EXIT_SMI:
12727	case VMX_EXIT_ERR_MSR_LOAD:
12728	case VMX_EXIT_ERR_MACHINE_CHECK:
12729	case VMX_EXIT_PML_FULL:
12730	case VMX_EXIT_RSM:
12731	default:
12732	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12733	}
12734	}
12735	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12736
12737
12738	/** @name VM-exit helpers.
12739	* @{
12740	*/
12741	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12742	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12743	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12744
12745	/** Macro for VM-exits called unexpectedly. */
12746	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
12747	do { \
12748	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
12749	return VERR_VMX_UNEXPECTED_EXIT; \
12750	} while (0)
12751
12752	#ifdef VBOX_STRICT
12753	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12754	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12755	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12756
12757	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12758	do { \
12759	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12760	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12761	} while (0)
12762
12763	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12764	do { \
12765	AssertPtr((a_pVCpu)); \
12766	AssertPtr((a_pVmxTransient)); \
12767	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12768	Assert((a_pVmxTransient)->pVmcsInfo); \
12769	Assert(ASMIntAreEnabled()); \
12770	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12771	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12772	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
12773	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12774	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12775	HMVMX_ASSERT_PREEMPT_CPUID(); \
12776	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12777	} while (0)
12778
12779	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12780	do { \
12781	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12782	Assert((a_pVmxTransient)->fIsNestedGuest); \
12783	} while (0)
12784
12785	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12786	do { \
12787	Log4Func(("\n")); \
12788	} while (0)
12789	#else
12790	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12791	do { \
12792	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12793	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12794	} while (0)
12795
12796	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12797	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12798
12799	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12800	#endif
12801
12802	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12803	/** Macro that does the necessary privilege checks and intercepted VM-exits for
12804	* guests that attempted to execute a VMX instruction. */
12805	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
12806	do \
12807	{ \
12808	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
12809	if (rcStrictTmp == VINF_SUCCESS) \
12810	{ /* likely */ } \
12811	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12812	{ \
12813	Assert((a_pVCpu)->hm.s.Event.fPending); \
12814	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
12815	return VINF_SUCCESS; \
12816	} \
12817	else \
12818	{ \
12819	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
12820	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
12821	} \
12822	} while (0)
12823
12824	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
12825	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
12826	do \
12827	{ \
12828	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
12829	(a_pGCPtrEffAddr)); \
12830	if (rcStrictTmp == VINF_SUCCESS) \
12831	{ /* likely */ } \
12832	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12833	{ \
12834	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
12835	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
12836	NOREF(uXcptTmp); \
12837	return VINF_SUCCESS; \
12838	} \
12839	else \
12840	{ \
12841	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
12842	return rcStrictTmp; \
12843	} \
12844	} while (0)
12845	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12846
12847
12848	/**
12849	* Advances the guest RIP by the specified number of bytes.
12850	*
12851	* @param pVCpu The cross context virtual CPU structure.
12852	* @param cbInstr Number of bytes to advance the RIP by.
12853	*
12854	* @remarks No-long-jump zone!!!
12855	*/
12856	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
12857	{
12858	/* Advance the RIP. */
12859	pVCpu->cpum.GstCtx.rip += cbInstr;
12860	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12861
12862	/* Update interrupt inhibition. */
12863	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12864	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12865	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12866	}
12867
12868
12869	/**
12870	* Advances the guest RIP after reading it from the VMCS.
12871	*
12872	* @returns VBox status code, no informational status codes.
12873	* @param pVCpu The cross context virtual CPU structure.
12874	* @param pVmxTransient The VMX-transient structure.
12875	*
12876	* @remarks No-long-jump zone!!!
12877	*/
12878	static int hmR0VmxAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12879	{
12880	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12881	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12882	AssertRCReturn(rc, rc);
12883
12884	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
12885	return VINF_SUCCESS;
12886	}
12887
12888
12889	/**
12890	* Handle a condition that occurred while delivering an event through the guest or
12891	* nested-guest IDT.
12892	*
12893	* @returns Strict VBox status code (i.e. informational status codes too).
12894	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12895	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12896	* to continue execution of the guest which will delivery the \#DF.
12897	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12898	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12899	*
12900	* @param pVCpu The cross context virtual CPU structure.
12901	* @param pVmxTransient The VMX-transient structure.
12902	*
12903	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
12904	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
12905	* is due to an EPT violation, PML full or SPP-related event.
12906	*
12907	* @remarks No-long-jump zone!!!
12908	*/
12909	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12910	{
12911	Assert(!pVCpu->hm.s.Event.fPending);
12912	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
12913	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
12914	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
12915	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
12916	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
12917
12918	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12919	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12920	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
12921	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12922	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
12923	{
12924	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
12925	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
12926
12927	/*
12928	* If the event was a software interrupt (generated with INT n) or a software exception
12929	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12930	* can handle the VM-exit and continue guest execution which will re-execute the
12931	* instruction rather than re-injecting the exception, as that can cause premature
12932	* trips to ring-3 before injection and involve TRPM which currently has no way of
12933	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12934	* the problem).
12935	*/
12936	IEMXCPTRAISE enmRaise;
12937	IEMXCPTRAISEINFO fRaiseInfo;
12938	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12939	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12940	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12941	{
12942	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12943	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12944	}
12945	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
12946	{
12947	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
12948	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12949	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
12950
12951	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12952	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12953
12954	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12955
12956	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12957	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12958	{
12959	pVmxTransient->fVectoringPF = true;
12960	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12961	}
12962	}
12963	else
12964	{
12965	/*
12966	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12967	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12968	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12969	*/
12970	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12971	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12972	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12973	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12974	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12975	}
12976
12977	/*
12978	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12979	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12980	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12981	* subsequent VM-entry would fail, see @bugref{7445}.
12982	*
12983	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
12984	*/
12985	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12986	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
12987	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
12988	&& CPUMIsGuestNmiBlocking(pVCpu))
12989	{
12990	CPUMSetGuestNmiBlocking(pVCpu, false);
12991	}
12992
12993	switch (enmRaise)
12994	{
12995	case IEMXCPTRAISE_CURRENT_XCPT:
12996	{
12997	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
12998	Assert(rcStrict == VINF_SUCCESS);
12999	break;
13000	}
13001
13002	case IEMXCPTRAISE_PREV_EVENT:
13003	{
13004	uint32_t u32ErrCode;
13005	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13006	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13007	else
13008	u32ErrCode = 0;
13009
13010	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13011	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13012	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13013	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13014
13015	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13016	pVCpu->hm.s.Event.u32ErrCode));
13017	Assert(rcStrict == VINF_SUCCESS);
13018	break;
13019	}
13020
13021	case IEMXCPTRAISE_REEXEC_INSTR:
13022	Assert(rcStrict == VINF_SUCCESS);
13023	break;
13024
13025	case IEMXCPTRAISE_DOUBLE_FAULT:
13026	{
13027	/*
13028	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13029	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13030	*/
13031	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13032	{
13033	pVmxTransient->fVectoringDoublePF = true;
13034	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13035	pVCpu->cpum.GstCtx.cr2));
13036	rcStrict = VINF_SUCCESS;
13037	}
13038	else
13039	{
13040	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13041	hmR0VmxSetPendingXcptDF(pVCpu);
13042	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13043	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13044	rcStrict = VINF_HM_DOUBLE_FAULT;
13045	}
13046	break;
13047	}
13048
13049	case IEMXCPTRAISE_TRIPLE_FAULT:
13050	{
13051	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13052	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13053	rcStrict = VINF_EM_RESET;
13054	break;
13055	}
13056
13057	case IEMXCPTRAISE_CPU_HANG:
13058	{
13059	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13060	rcStrict = VERR_EM_GUEST_CPU_HANG;
13061	break;
13062	}
13063
13064	default:
13065	{
13066	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13067	rcStrict = VERR_VMX_IPE_2;
13068	break;
13069	}
13070	}
13071	}
13072	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13073	&& !CPUMIsGuestNmiBlocking(pVCpu))
13074	{
13075	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13076	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13077	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13078	{
13079	/*
13080	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13081	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13082	* that NMIs remain blocked until the IRET execution is completed.
13083	*
13084	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13085	*/
13086	CPUMSetGuestNmiBlocking(pVCpu, true);
13087	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13088	}
13089	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13090	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13091	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13092	{
13093	/*
13094	* Execution of IRET caused an EPT violation, page-modification log-full event or
13095	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13096	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13097	* that NMIs remain blocked until the IRET execution is completed.
13098	*
13099	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13100	*/
13101	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13102	{
13103	CPUMSetGuestNmiBlocking(pVCpu, true);
13104	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13105	}
13106	}
13107	}
13108
13109	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13110	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13111	return rcStrict;
13112	}
13113
13114
13115	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13116	/**
13117	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13118	* guest attempting to execute a VMX instruction.
13119	*
13120	* @returns Strict VBox status code (i.e. informational status codes too).
13121	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13122	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13123	*
13124	* @param pVCpu The cross context virtual CPU structure.
13125	* @param uExitReason The VM-exit reason.
13126	*
13127	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13128	* @remarks No-long-jump zone!!!
13129	*/
13130	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
13131	{
13132	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13133	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13134
13135	/*
13136	* The physical CPU would have already checked the CPU mode/code segment.
13137	* We shall just assert here for paranoia.
13138	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13139	*/
13140	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13141	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13142	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13143
13144	if (uExitReason == VMX_EXIT_VMXON)
13145	{
13146	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13147
13148	/*
13149	* We check CR4.VMXE because it is required to be always set while in VMX operation
13150	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13151	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13152	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13153	*/
13154	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13155	{
13156	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13157	hmR0VmxSetPendingXcptUD(pVCpu);
13158	return VINF_HM_PENDING_XCPT;
13159	}
13160	}
13161	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13162	{
13163	/*
13164	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13165	* (other than VMXON), we need to raise a #UD.
13166	*/
13167	Log4Func(("Not in VMX root mode -> #UD\n"));
13168	hmR0VmxSetPendingXcptUD(pVCpu);
13169	return VINF_HM_PENDING_XCPT;
13170	}
13171
13172	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13173	return VINF_SUCCESS;
13174	}
13175
13176
13177	/**
13178	* Decodes the memory operand of an instruction that caused a VM-exit.
13179	*
13180	* The Exit qualification field provides the displacement field for memory
13181	* operand instructions, if any.
13182	*
13183	* @returns Strict VBox status code (i.e. informational status codes too).
13184	* @retval VINF_SUCCESS if the operand was successfully decoded.
13185	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13186	* operand.
13187	* @param pVCpu The cross context virtual CPU structure.
13188	* @param uExitInstrInfo The VM-exit instruction information field.
13189	* @param enmMemAccess The memory operand's access type (read or write).
13190	* @param GCPtrDisp The instruction displacement field, if any. For
13191	* RIP-relative addressing pass RIP + displacement here.
13192	* @param pGCPtrMem Where to store the effective destination memory address.
13193	*
13194	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13195	* virtual-8086 mode hence skips those checks while verifying if the
13196	* segment is valid.
13197	*/
13198	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13199	PRTGCPTR pGCPtrMem)
13200	{
13201	Assert(pGCPtrMem);
13202	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13203	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13204	\| CPUMCTX_EXTRN_CR0);
13205
13206	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13207	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13208	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13209
13210	VMXEXITINSTRINFO ExitInstrInfo;
13211	ExitInstrInfo.u = uExitInstrInfo;
13212	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13213	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13214	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13215	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13216	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13217	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13218	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13219	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13220	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13221
13222	/*
13223	* Validate instruction information.
13224	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13225	*/
13226	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13227	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13228	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13229	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13230	AssertLogRelMsgReturn(fIsMemOperand,
13231	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13232
13233	/*
13234	* Compute the complete effective address.
13235	*
13236	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13237	* See AMD spec. 4.5.2 "Segment Registers".
13238	*/
13239	RTGCPTR GCPtrMem = GCPtrDisp;
13240	if (fBaseRegValid)
13241	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13242	if (fIdxRegValid)
13243	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13244
13245	RTGCPTR const GCPtrOff = GCPtrMem;
13246	if ( !fIsLongMode
13247	\|\| iSegReg >= X86_SREG_FS)
13248	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13249	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13250
13251	/*
13252	* Validate effective address.
13253	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13254	*/
13255	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13256	Assert(cbAccess > 0);
13257	if (fIsLongMode)
13258	{
13259	if (X86_IS_CANONICAL(GCPtrMem))
13260	{
13261	*pGCPtrMem = GCPtrMem;
13262	return VINF_SUCCESS;
13263	}
13264
13265	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13266	* "Data Limit Checks in 64-bit Mode". */
13267	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13268	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13269	return VINF_HM_PENDING_XCPT;
13270	}
13271
13272	/*
13273	* This is a watered down version of iemMemApplySegment().
13274	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13275	* and segment CPL/DPL checks are skipped.
13276	*/
13277	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13278	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13279	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13280
13281	/* Check if the segment is present and usable. */
13282	if ( pSel->Attr.n.u1Present
13283	&& !pSel->Attr.n.u1Unusable)
13284	{
13285	Assert(pSel->Attr.n.u1DescType);
13286	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13287	{
13288	/* Check permissions for the data segment. */
13289	if ( enmMemAccess == VMXMEMACCESS_WRITE
13290	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13291	{
13292	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13293	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13294	return VINF_HM_PENDING_XCPT;
13295	}
13296
13297	/* Check limits if it's a normal data segment. */
13298	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13299	{
13300	if ( GCPtrFirst32 > pSel->u32Limit
13301	\|\| GCPtrLast32 > pSel->u32Limit)
13302	{
13303	Log4Func(("Data segment limit exceeded. "
13304	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13305	GCPtrLast32, pSel->u32Limit));
13306	if (iSegReg == X86_SREG_SS)
13307	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13308	else
13309	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13310	return VINF_HM_PENDING_XCPT;
13311	}
13312	}
13313	else
13314	{
13315	/* Check limits if it's an expand-down data segment.
13316	Note! The upper boundary is defined by the B bit, not the G bit! */
13317	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13318	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13319	{
13320	Log4Func(("Expand-down data segment limit exceeded. "
13321	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13322	GCPtrLast32, pSel->u32Limit));
13323	if (iSegReg == X86_SREG_SS)
13324	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13325	else
13326	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13327	return VINF_HM_PENDING_XCPT;
13328	}
13329	}
13330	}
13331	else
13332	{
13333	/* Check permissions for the code segment. */
13334	if ( enmMemAccess == VMXMEMACCESS_WRITE
13335	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13336	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13337	{
13338	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13339	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13340	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13341	return VINF_HM_PENDING_XCPT;
13342	}
13343
13344	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13345	if ( GCPtrFirst32 > pSel->u32Limit
13346	\|\| GCPtrLast32 > pSel->u32Limit)
13347	{
13348	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13349	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13350	if (iSegReg == X86_SREG_SS)
13351	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13352	else
13353	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13354	return VINF_HM_PENDING_XCPT;
13355	}
13356	}
13357	}
13358	else
13359	{
13360	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13361	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13362	return VINF_HM_PENDING_XCPT;
13363	}
13364
13365	*pGCPtrMem = GCPtrMem;
13366	return VINF_SUCCESS;
13367	}
13368	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13369
13370
13371	/**
13372	* VM-exit helper for LMSW.
13373	*/
13374	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13375	{
13376	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13377	AssertRCReturn(rc, rc);
13378
13379	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13380	AssertMsg( rcStrict == VINF_SUCCESS
13381	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13382
13383	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13384	if (rcStrict == VINF_IEM_RAISED_XCPT)
13385	{
13386	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13387	rcStrict = VINF_SUCCESS;
13388	}
13389
13390	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13391	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13392	return rcStrict;
13393	}
13394
13395
13396	/**
13397	* VM-exit helper for CLTS.
13398	*/
13399	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13400	{
13401	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13402	AssertRCReturn(rc, rc);
13403
13404	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13405	AssertMsg( rcStrict == VINF_SUCCESS
13406	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13407
13408	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13409	if (rcStrict == VINF_IEM_RAISED_XCPT)
13410	{
13411	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13412	rcStrict = VINF_SUCCESS;
13413	}
13414
13415	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13416	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13417	return rcStrict;
13418	}
13419
13420
13421	/**
13422	* VM-exit helper for MOV from CRx (CRx read).
13423	*/
13424	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13425	{
13426	Assert(iCrReg < 16);
13427	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13428
13429	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13430	AssertRCReturn(rc, rc);
13431
13432	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13433	AssertMsg( rcStrict == VINF_SUCCESS
13434	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13435
13436	if (iGReg == X86_GREG_xSP)
13437	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13438	else
13439	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13440	#ifdef VBOX_WITH_STATISTICS
13441	switch (iCrReg)
13442	{
13443	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13444	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13445	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13446	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13447	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13448	}
13449	#endif
13450	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13451	return rcStrict;
13452	}
13453
13454
13455	/**
13456	* VM-exit helper for MOV to CRx (CRx write).
13457	*/
13458	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13459	{
13460	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13461	AssertRCReturn(rc, rc);
13462
13463	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13464	AssertMsg( rcStrict == VINF_SUCCESS
13465	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13466	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13467
13468	switch (iCrReg)
13469	{
13470	case 0:
13471	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
13472	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13473	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13474	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13475	break;
13476
13477	case 2:
13478	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13479	/* Nothing to do here, CR2 it's not part of the VMCS. */
13480	break;
13481
13482	case 3:
13483	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13484	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13485	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13486	break;
13487
13488	case 4:
13489	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13490	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13491	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13492	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13493	break;
13494
13495	case 8:
13496	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13497	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13498	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13499	break;
13500
13501	default:
13502	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13503	break;
13504	}
13505
13506	if (rcStrict == VINF_IEM_RAISED_XCPT)
13507	{
13508	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13509	rcStrict = VINF_SUCCESS;
13510	}
13511	return rcStrict;
13512	}
13513
13514
13515	/**
13516	* VM-exit exception handler for \#PF (Page-fault exception).
13517	*
13518	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13519	*/
13520	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13521	{
13522	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13523	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13524	hmR0VmxReadExitQualVmcs(pVmxTransient);
13525
13526	if (!pVM->hm.s.fNestedPaging)
13527	{ /* likely */ }
13528	else
13529	{
13530	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13531	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13532	#endif
13533	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13534	if (!pVmxTransient->fVectoringDoublePF)
13535	{
13536	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13537	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13538	}
13539	else
13540	{
13541	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13542	Assert(!pVmxTransient->fIsNestedGuest);
13543	hmR0VmxSetPendingXcptDF(pVCpu);
13544	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13545	}
13546	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13547	return VINF_SUCCESS;
13548	}
13549
13550	Assert(!pVmxTransient->fIsNestedGuest);
13551
13552	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13553	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13554	if (pVmxTransient->fVectoringPF)
13555	{
13556	Assert(pVCpu->hm.s.Event.fPending);
13557	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13558	}
13559
13560	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13561	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13562	AssertRCReturn(rc, rc);
13563
13564	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
13565	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
13566
13567	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13568	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13569
13570	Log4Func(("#PF: rc=%Rrc\n", rc));
13571	if (rc == VINF_SUCCESS)
13572	{
13573	/*
13574	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13575	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13576	*/
13577	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13578	TRPMResetTrap(pVCpu);
13579	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13580	return rc;
13581	}
13582
13583	if (rc == VINF_EM_RAW_GUEST_TRAP)
13584	{
13585	if (!pVmxTransient->fVectoringDoublePF)
13586	{
13587	/* It's a guest page fault and needs to be reflected to the guest. */
13588	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
13589	TRPMResetTrap(pVCpu);
13590	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13591	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13592	uGstErrorCode, pVmxTransient->uExitQual);
13593	}
13594	else
13595	{
13596	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13597	TRPMResetTrap(pVCpu);
13598	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13599	hmR0VmxSetPendingXcptDF(pVCpu);
13600	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13601	}
13602
13603	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13604	return VINF_SUCCESS;
13605	}
13606
13607	TRPMResetTrap(pVCpu);
13608	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13609	return rc;
13610	}
13611
13612
13613	/**
13614	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13615	*
13616	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13617	*/
13618	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13619	{
13620	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13621	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13622
13623	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13624	AssertRCReturn(rc, rc);
13625
13626	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13627	{
13628	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13629	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13630
13631	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13632	* provides VM-exit instruction length. If this causes problem later,
13633	* disassemble the instruction like it's done on AMD-V. */
13634	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13635	AssertRCReturn(rc2, rc2);
13636	return rc;
13637	}
13638
13639	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
13640	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13641	return VINF_SUCCESS;
13642	}
13643
13644
13645	/**
13646	* VM-exit exception handler for \#BP (Breakpoint exception).
13647	*
13648	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13649	*/
13650	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13651	{
13652	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13653	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13654
13655	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13656	AssertRCReturn(rc, rc);
13657
13658	if (!pVmxTransient->fIsNestedGuest)
13659	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13660	else
13661	rc = VINF_EM_RAW_GUEST_TRAP;
13662
13663	if (rc == VINF_EM_RAW_GUEST_TRAP)
13664	{
13665	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13666	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13667	rc = VINF_SUCCESS;
13668	}
13669
13670	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
13671	return rc;
13672	}
13673
13674
13675	/**
13676	* VM-exit exception handler for \#AC (Alignment-check exception).
13677	*
13678	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13679	*/
13680	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13681	{
13682	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13683	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13684
13685	/* Re-inject it. We'll detect any nesting before getting here. */
13686	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13687	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13688	return VINF_SUCCESS;
13689	}
13690
13691
13692	/**
13693	* VM-exit exception handler for \#DB (Debug exception).
13694	*
13695	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13696	*/
13697	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13698	{
13699	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13700	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13701
13702	/*
13703	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13704	*/
13705	hmR0VmxReadExitQualVmcs(pVmxTransient);
13706
13707	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13708	uint64_t const uDR6 = X86_DR6_INIT_VAL
13709	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13710	\| X86_DR6_BD \| X86_DR6_BS));
13711
13712	int rc;
13713	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13714	if (!pVmxTransient->fIsNestedGuest)
13715	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13716	else
13717	rc = VINF_EM_RAW_GUEST_TRAP;
13718	Log6Func(("rc=%Rrc\n", rc));
13719	if (rc == VINF_EM_RAW_GUEST_TRAP)
13720	{
13721	/*
13722	* The exception was for the guest. Update DR6, DR7.GD and
13723	* IA32_DEBUGCTL.LBR before forwarding it.
13724	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
13725	*/
13726	VMMRZCallRing3Disable(pVCpu);
13727	HM_DISABLE_PREEMPT(pVCpu);
13728
13729	pCtx->dr[6] &= ~X86_DR6_B_MASK;
13730	pCtx->dr[6] \|= uDR6;
13731	if (CPUMIsGuestDebugStateActive(pVCpu))
13732	ASMSetDR6(pCtx->dr[6]);
13733
13734	HM_RESTORE_PREEMPT();
13735	VMMRZCallRing3Enable(pVCpu);
13736
13737	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
13738	AssertRCReturn(rc, rc);
13739
13740	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13741	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
13742
13743	/* Paranoia. */
13744	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
13745	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
13746
13747	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
13748	AssertRC(rc);
13749
13750	/*
13751	* Raise #DB in the guest.
13752	*
13753	* It is important to reflect exactly what the VM-exit gave us (preserving the
13754	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13755	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13756	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13757	*
13758	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13759	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13760	*/
13761	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13762	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13763	return VINF_SUCCESS;
13764	}
13765
13766	/*
13767	* Not a guest trap, must be a hypervisor related debug event then.
13768	* Update DR6 in case someone is interested in it.
13769	*/
13770	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13771	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13772	CPUMSetHyperDR6(pVCpu, uDR6);
13773
13774	return rc;
13775	}
13776
13777
13778	/**
13779	* Hacks its way around the lovely mesa driver's backdoor accesses.
13780	*
13781	* @sa hmR0SvmHandleMesaDrvGp.
13782	*/
13783	static int hmR0VmxHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13784	{
13785	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
13786	RT_NOREF(pCtx);
13787
13788	/* For now we'll just skip the instruction. */
13789	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13790	}
13791
13792
13793	/**
13794	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
13795	* backdoor logging w/o checking what it is running inside.
13796	*
13797	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
13798	* backdoor port and magic numbers loaded in registers.
13799	*
13800	* @returns true if it is, false if it isn't.
13801	* @sa hmR0SvmIsMesaDrvGp.
13802	*/
13803	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13804	{
13805	/* 0xed: IN eAX,dx */
13806	uint8_t abInstr[1];
13807	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
13808	return false;
13809
13810	/* Check that it is #GP(0). */
13811	if (pVmxTransient->uExitIntErrorCode != 0)
13812	return false;
13813
13814	/* Check magic and port. */
13815	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
13816	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
13817	if (pCtx->rax != UINT32_C(0x564d5868))
13818	return false;
13819	if (pCtx->dx != UINT32_C(0x5658))
13820	return false;
13821
13822	/* Flat ring-3 CS. */
13823	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
13824	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
13825	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
13826	if (pCtx->cs.Attr.n.u2Dpl != 3)
13827	return false;
13828	if (pCtx->cs.u64Base != 0)
13829	return false;
13830
13831	/* Check opcode. */
13832	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
13833	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
13834	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
13835	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
13836	if (RT_FAILURE(rc))
13837	return false;
13838	if (abInstr[0] != 0xed)
13839	return false;
13840
13841	return true;
13842	}
13843
13844
13845	/**
13846	* VM-exit exception handler for \#GP (General-protection exception).
13847	*
13848	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13849	*/
13850	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13851	{
13852	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13853	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13854
13855	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13856	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13857	if (pVmcsInfo->RealMode.fRealOnV86Active)
13858	{ /* likely */ }
13859	else
13860	{
13861	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13862	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
13863	#endif
13864	/*
13865	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
13866	* executing a nested-guest, reflect #GP to the guest or nested-guest.
13867	*/
13868	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13869	AssertRCReturn(rc, rc);
13870	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13871	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13872
13873	if ( pVmxTransient->fIsNestedGuest
13874	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
13875	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
13876	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13877	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13878	else
13879	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
13880	return rc;
13881	}
13882
13883	Assert(CPUMIsGuestInRealModeEx(pCtx));
13884	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13885	Assert(!pVmxTransient->fIsNestedGuest);
13886
13887	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13888	AssertRCReturn(rc, rc);
13889
13890	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13891	if (rcStrict == VINF_SUCCESS)
13892	{
13893	if (!CPUMIsGuestInRealModeEx(pCtx))
13894	{
13895	/*
13896	* The guest is no longer in real-mode, check if we can continue executing the
13897	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13898	*/
13899	pVmcsInfo->RealMode.fRealOnV86Active = false;
13900	if (HMCanExecuteVmxGuest(pVCpu->pVMR0, pVCpu, pCtx))
13901	{
13902	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
13903	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13904	}
13905	else
13906	{
13907	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13908	rcStrict = VINF_EM_RESCHEDULE;
13909	}
13910	}
13911	else
13912	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13913	}
13914	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13915	{
13916	rcStrict = VINF_SUCCESS;
13917	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13918	}
13919	return VBOXSTRICTRC_VAL(rcStrict);
13920	}
13921
13922
13923	/**
13924	* VM-exit exception handler wrapper for all other exceptions that are not handled
13925	* by a specific handler.
13926	*
13927	* This simply re-injects the exception back into the VM without any special
13928	* processing.
13929	*
13930	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13931	*/
13932	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13933	{
13934	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13935
13936	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13937	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13938	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
13939	("uVector=%#x u32XcptBitmap=%#X32\n",
13940	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
13941	NOREF(pVmcsInfo);
13942	#endif
13943
13944	/*
13945	* Re-inject the exception into the guest. This cannot be a double-fault condition which
13946	* would have been handled while checking exits due to event delivery.
13947	*/
13948	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13949
13950	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13951	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13952	AssertRCReturn(rc, rc);
13953	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13954	#endif
13955
13956	#ifdef VBOX_WITH_STATISTICS
13957	switch (uVector)
13958	{
13959	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
13960	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
13961	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
13962	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
13963	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
13964	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
13965	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
13966	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
13967	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
13968	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
13969	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
13970	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
13971	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
13972	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
13973	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
13974	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
13975	default:
13976	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13977	break;
13978	}
13979	#endif
13980
13981	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
13982	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
13983	NOREF(uVector);
13984
13985	/* Re-inject the original exception into the guest. */
13986	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13987	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13988	return VINF_SUCCESS;
13989	}
13990
13991
13992	/**
13993	* VM-exit exception handler for all exceptions (except NMIs!).
13994	*
13995	* @remarks This may be called for both guests and nested-guests. Take care to not
13996	* make assumptions and avoid doing anything that is not relevant when
13997	* executing a nested-guest (e.g., Mesa driver hacks).
13998	*/
13999	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14000	{
14001	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
14002
14003	/*
14004	* If this VM-exit occurred while delivering an event through the guest IDT, take
14005	* action based on the return code and additional hints (e.g. for page-faults)
14006	* that will be updated in the VMX transient structure.
14007	*/
14008	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14009	if (rcStrict == VINF_SUCCESS)
14010	{
14011	/*
14012	* If an exception caused a VM-exit due to delivery of an event, the original
14013	* event may have to be re-injected into the guest. We shall reinject it and
14014	* continue guest execution. However, page-fault is a complicated case and
14015	* needs additional processing done in hmR0VmxExitXcptPF().
14016	*/
14017	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14018	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14019	if ( !pVCpu->hm.s.Event.fPending
14020	\|\| uVector == X86_XCPT_PF)
14021	{
14022	switch (uVector)
14023	{
14024	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
14025	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
14026	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
14027	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
14028	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
14029	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
14030	default:
14031	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
14032	}
14033	}
14034	/* else: inject pending event before resuming guest execution. */
14035	}
14036	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
14037	{
14038	Assert(pVCpu->hm.s.Event.fPending);
14039	rcStrict = VINF_SUCCESS;
14040	}
14041
14042	return rcStrict;
14043	}
14044	/** @} */
14045
14046
14047	/** @name VM-exit handlers.
14048	* @{
14049	*/
14050	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14051	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14052	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14053
14054	/**
14055	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14056	*/
14057	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14058	{
14059	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14060	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14061	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14062	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14063	return VINF_SUCCESS;
14064	return VINF_EM_RAW_INTERRUPT;
14065	}
14066
14067
14068	/**
14069	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14070	* VM-exit.
14071	*/
14072	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14073	{
14074	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14075	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14076
14077	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14078
14079	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14080	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14081	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14082
14083	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14084	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14085	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14086	NOREF(pVmcsInfo);
14087
14088	VBOXSTRICTRC rcStrict;
14089	switch (uExitIntType)
14090	{
14091	/*
14092	* Host physical NMIs:
14093	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14094	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14095	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14096	*
14097	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14098	* See Intel spec. 27.5.5 "Updating Non-Register State".
14099	*/
14100	case VMX_EXIT_INT_INFO_TYPE_NMI:
14101	{
14102	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14103	break;
14104	}
14105
14106	/*
14107	* Privileged software exceptions (#DB from ICEBP),
14108	* Software exceptions (#BP and #OF),
14109	* Hardware exceptions:
14110	* Process the required exceptions and resume guest execution if possible.
14111	*/
14112	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14113	Assert(uVector == X86_XCPT_DB);
14114	RT_FALL_THRU();
14115	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14116	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14117	RT_FALL_THRU();
14118	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14119	{
14120	NOREF(uVector);
14121	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14122	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14123	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14124	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14125
14126	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14127	break;
14128	}
14129
14130	default:
14131	{
14132	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14133	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14134	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14135	break;
14136	}
14137	}
14138
14139	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14140	return rcStrict;
14141	}
14142
14143
14144	/**
14145	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14146	*/
14147	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14148	{
14149	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14150
14151	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14152	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14153	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14154
14155	/* Evaluate and deliver pending events and resume guest execution. */
14156	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14157	return VINF_SUCCESS;
14158	}
14159
14160
14161	/**
14162	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14163	*/
14164	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14165	{
14166	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14167
14168	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14169	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14170	{
14171	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14172	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14173	}
14174
14175	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14176
14177	/*
14178	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14179	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14180	*/
14181	uint32_t fIntrState;
14182	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14183	AssertRC(rc);
14184	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14185	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14186	{
14187	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14188	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14189
14190	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14191	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14192	AssertRC(rc);
14193	}
14194
14195	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14196	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14197
14198	/* Evaluate and deliver pending events and resume guest execution. */
14199	return VINF_SUCCESS;
14200	}
14201
14202
14203	/**
14204	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14205	*/
14206	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14207	{
14208	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14209	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14210	}
14211
14212
14213	/**
14214	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14215	*/
14216	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14217	{
14218	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14219	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14220	}
14221
14222
14223	/**
14224	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14225	*/
14226	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14227	{
14228	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14229
14230	/*
14231	* Get the state we need and update the exit history entry.
14232	*/
14233	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14234	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14235
14236	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14237	AssertRCReturn(rc, rc);
14238
14239	VBOXSTRICTRC rcStrict;
14240	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14241	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14242	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14243	if (!pExitRec)
14244	{
14245	/*
14246	* Regular CPUID instruction execution.
14247	*/
14248	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
14249	if (rcStrict == VINF_SUCCESS)
14250	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14251	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14252	{
14253	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14254	rcStrict = VINF_SUCCESS;
14255	}
14256	}
14257	else
14258	{
14259	/*
14260	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14261	*/
14262	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14263	AssertRCReturn(rc2, rc2);
14264
14265	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14266	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14267
14268	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14269	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14270
14271	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14272	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14273	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14274	}
14275	return rcStrict;
14276	}
14277
14278
14279	/**
14280	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14281	*/
14282	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14283	{
14284	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14285
14286	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14287	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14288	AssertRCReturn(rc, rc);
14289
14290	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14291	return VINF_EM_RAW_EMULATE_INSTR;
14292
14293	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14294	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14295	}
14296
14297
14298	/**
14299	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14300	*/
14301	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14302	{
14303	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14304
14305	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14306	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14307	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14308	AssertRCReturn(rc, rc);
14309
14310	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
14311	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14312	{
14313	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14314	we must reset offsetting on VM-entry. See @bugref{6634}. */
14315	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14316	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14317	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14318	}
14319	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14320	{
14321	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14322	rcStrict = VINF_SUCCESS;
14323	}
14324	return rcStrict;
14325	}
14326
14327
14328	/**
14329	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14330	*/
14331	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14332	{
14333	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14334
14335	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14336	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14337	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14338	AssertRCReturn(rc, rc);
14339
14340	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
14341	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14342	{
14343	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14344	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14345	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14346	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14347	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14348	}
14349	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14350	{
14351	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14352	rcStrict = VINF_SUCCESS;
14353	}
14354	return rcStrict;
14355	}
14356
14357
14358	/**
14359	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14360	*/
14361	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14362	{
14363	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14364
14365	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14366	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14367	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14368	AssertRCReturn(rc, rc);
14369
14370	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14371	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14372	if (RT_LIKELY(rc == VINF_SUCCESS))
14373	{
14374	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14375	Assert(pVmxTransient->cbExitInstr == 2);
14376	}
14377	else
14378	{
14379	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14380	rc = VERR_EM_INTERPRETER;
14381	}
14382	return rc;
14383	}
14384
14385
14386	/**
14387	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14388	*/
14389	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14390	{
14391	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14392
14393	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14394	if (EMAreHypercallInstructionsEnabled(pVCpu))
14395	{
14396	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14397	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14398	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14399	AssertRCReturn(rc, rc);
14400
14401	/* Perform the hypercall. */
14402	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14403	if (rcStrict == VINF_SUCCESS)
14404	{
14405	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14406	AssertRCReturn(rc, rc);
14407	}
14408	else
14409	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14410	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14411	\|\| RT_FAILURE(rcStrict));
14412
14413	/* If the hypercall changes anything other than guest's general-purpose registers,
14414	we would need to reload the guest changed bits here before VM-entry. */
14415	}
14416	else
14417	Log4Func(("Hypercalls not enabled\n"));
14418
14419	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14420	if (RT_FAILURE(rcStrict))
14421	{
14422	hmR0VmxSetPendingXcptUD(pVCpu);
14423	rcStrict = VINF_SUCCESS;
14424	}
14425
14426	return rcStrict;
14427	}
14428
14429
14430	/**
14431	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14432	*/
14433	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14434	{
14435	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14436	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14437
14438	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14439	hmR0VmxReadExitQualVmcs(pVmxTransient);
14440	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14441	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14442	AssertRCReturn(rc, rc);
14443
14444	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
14445
14446	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14447	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14448	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14449	{
14450	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14451	rcStrict = VINF_SUCCESS;
14452	}
14453	else
14454	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14455	VBOXSTRICTRC_VAL(rcStrict)));
14456	return rcStrict;
14457	}
14458
14459
14460	/**
14461	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14462	*/
14463	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14464	{
14465	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14466
14467	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14468	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14469	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14470	AssertRCReturn(rc, rc);
14471
14472	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
14473	if (rcStrict == VINF_SUCCESS)
14474	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14475	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14476	{
14477	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14478	rcStrict = VINF_SUCCESS;
14479	}
14480
14481	return rcStrict;
14482	}
14483
14484
14485	/**
14486	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14487	*/
14488	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14489	{
14490	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14491
14492	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14493	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14494	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14495	AssertRCReturn(rc, rc);
14496
14497	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
14498	if (RT_SUCCESS(rcStrict))
14499	{
14500	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14501	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14502	rcStrict = VINF_SUCCESS;
14503	}
14504
14505	return rcStrict;
14506	}
14507
14508
14509	/**
14510	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14511	* VM-exit.
14512	*/
14513	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14514	{
14515	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14516	return VINF_EM_RESET;
14517	}
14518
14519
14520	/**
14521	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14522	*/
14523	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14524	{
14525	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14526
14527	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14528	AssertRCReturn(rc, rc);
14529
14530	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14531	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14532	rc = VINF_SUCCESS;
14533	else
14534	rc = VINF_EM_HALT;
14535
14536	if (rc != VINF_SUCCESS)
14537	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14538	return rc;
14539	}
14540
14541
14542	/**
14543	* VM-exit handler for instructions that result in a \#UD exception delivered to
14544	* the guest.
14545	*/
14546	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14547	{
14548	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14549	hmR0VmxSetPendingXcptUD(pVCpu);
14550	return VINF_SUCCESS;
14551	}
14552
14553
14554	/**
14555	* VM-exit handler for expiry of the VMX-preemption timer.
14556	*/
14557	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14558	{
14559	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14560
14561	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14562	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14563
14564	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14565	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14566	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14568	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14569	}
14570
14571
14572	/**
14573	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14574	*/
14575	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14576	{
14577	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14578
14579	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14580	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14581	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14582	AssertRCReturn(rc, rc);
14583
14584	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
14585	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14586	: HM_CHANGED_RAISED_XCPT_MASK);
14587
14588	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14589	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14590
14591	return rcStrict;
14592	}
14593
14594
14595	/**
14596	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14597	*/
14598	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14599	{
14600	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14601
14602	#if 1
14603	/** @todo Use VM-exit instruction information. */
14604	return VERR_EM_INTERPRETER;
14605	#else
14606	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14607	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14608	hmR0VmxReadExitQualVmcs(pVmxTransient);
14609	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
14610	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14611	AssertRCReturn(rc, rc);
14612
14613	/* Paranoia. Ensure this has a memory operand. */
14614	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
14615
14616	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
14617	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14618	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
14619	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
14620
14621	RTGCPTR GCPtrDesc;
14622	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
14623
14624	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
14625	GCPtrDesc, uType);
14626	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14627	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14628	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14629	{
14630	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14631	rcStrict = VINF_SUCCESS;
14632	}
14633	return rcStrict;
14634	#endif
14635	}
14636
14637
14638	/**
14639	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14640	* VM-exit.
14641	*/
14642	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14643	{
14644	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14645	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14646	AssertRCReturn(rc, rc);
14647
14648	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14649	if (RT_FAILURE(rc))
14650	return rc;
14651
14652	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14653	NOREF(uInvalidReason);
14654
14655	#ifdef VBOX_STRICT
14656	uint32_t fIntrState;
14657	uint64_t u64Val;
14658	hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14659	hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14660	hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14661
14662	Log4(("uInvalidReason %u\n", uInvalidReason));
14663	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14664	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14665	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14666
14667	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
14668	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14669	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
14670	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
14671	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
14672	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
14673	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
14674	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14675	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
14676	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
14677	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
14678	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14679	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14680	{
14681	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14682	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14683	}
14684	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
14685	#endif
14686
14687	return VERR_VMX_INVALID_GUEST_STATE;
14688	}
14689
14690	/**
14691	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14692	*/
14693	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14694	{
14695	/*
14696	* Cumulative notes of all recognized but unexpected VM-exits.
14697	*
14698	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
14699	* nested-paging is used.
14700	*
14701	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14702	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14703	* this function (and thereby stop VM execution) for handling such instructions.
14704	*
14705	*
14706	* VMX_EXIT_INIT_SIGNAL:
14707	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14708	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14709	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14710	*
14711	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14712	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14713	* See Intel spec. "23.8 Restrictions on VMX operation".
14714	*
14715	* VMX_EXIT_SIPI:
14716	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14717	* activity state is used. We don't make use of it as our guests don't have direct
14718	* access to the host local APIC.
14719	*
14720	* See Intel spec. 25.3 "Other Causes of VM-exits".
14721	*
14722	* VMX_EXIT_IO_SMI:
14723	* VMX_EXIT_SMI:
14724	* This can only happen if we support dual-monitor treatment of SMI, which can be
14725	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14726	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14727	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14728	*
14729	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14730	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14731	*
14732	* VMX_EXIT_ERR_MSR_LOAD:
14733	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14734	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14735	* execution.
14736	*
14737	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14738	*
14739	* VMX_EXIT_ERR_MACHINE_CHECK:
14740	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14741	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14742	* #MC exception abort class exception is raised. We thus cannot assume a
14743	* reasonable chance of continuing any sort of execution and we bail.
14744	*
14745	* See Intel spec. 15.1 "Machine-check Architecture".
14746	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14747	*
14748	* VMX_EXIT_PML_FULL:
14749	* VMX_EXIT_VIRTUALIZED_EOI:
14750	* VMX_EXIT_APIC_WRITE:
14751	* We do not currently support any of these features and thus they are all unexpected
14752	* VM-exits.
14753	*
14754	* VMX_EXIT_GDTR_IDTR_ACCESS:
14755	* VMX_EXIT_LDTR_TR_ACCESS:
14756	* VMX_EXIT_RDRAND:
14757	* VMX_EXIT_RSM:
14758	* VMX_EXIT_VMFUNC:
14759	* VMX_EXIT_ENCLS:
14760	* VMX_EXIT_RDSEED:
14761	* VMX_EXIT_XSAVES:
14762	* VMX_EXIT_XRSTORS:
14763	* VMX_EXIT_UMWAIT:
14764	* VMX_EXIT_TPAUSE:
14765	* These VM-exits are -not- caused unconditionally by execution of the corresponding
14766	* instruction. Any VM-exit for these instructions indicate a hardware problem,
14767	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14768	*
14769	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14770	*/
14771	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14772	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14773	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14774	}
14775
14776
14777	/**
14778	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14779	*/
14780	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14781	{
14782	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14783
14784	/** @todo Optimize this: We currently drag in the whole MSR state
14785	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14786	* MSRs required. That would require changes to IEM and possibly CPUM too.
14787	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14788	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14789	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14790	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14791	switch (idMsr)
14792	{
14793	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14794	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14795	}
14796
14797	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14798	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14799	AssertRCReturn(rc, rc);
14800
14801	Log4Func(("ecx=%#RX32\n", idMsr));
14802
14803	#ifdef VBOX_STRICT
14804	Assert(!pVmxTransient->fIsNestedGuest);
14805	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
14806	{
14807	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14808	&& idMsr != MSR_K6_EFER)
14809	{
14810	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14811	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14812	}
14813	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14814	{
14815	Assert(pVmcsInfo->pvMsrBitmap);
14816	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14817	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14818	{
14819	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14820	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14821	}
14822	}
14823	}
14824	#endif
14825
14826	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
14827	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14828	if (rcStrict == VINF_SUCCESS)
14829	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14830	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14831	{
14832	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14833	rcStrict = VINF_SUCCESS;
14834	}
14835	else
14836	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14837
14838	return rcStrict;
14839	}
14840
14841
14842	/**
14843	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14844	*/
14845	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14846	{
14847	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14848
14849	/** @todo Optimize this: We currently drag in the whole MSR state
14850	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14851	* MSRs required. That would require changes to IEM and possibly CPUM too.
14852	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14853	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14854	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14855
14856	/*
14857	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14858	* Although we don't need to fetch the base as it will be overwritten shortly, while
14859	* loading guest-state we would also load the entire segment register including limit
14860	* and attributes and thus we need to load them here.
14861	*/
14862	switch (idMsr)
14863	{
14864	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14865	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14866	}
14867
14868	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14869	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14870	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14871	AssertRCReturn(rc, rc);
14872
14873	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14874
14875	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
14876	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14877
14878	if (rcStrict == VINF_SUCCESS)
14879	{
14880	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14881
14882	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14883	if ( idMsr == MSR_IA32_APICBASE
14884	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14885	&& idMsr <= MSR_IA32_X2APIC_END))
14886	{
14887	/*
14888	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14889	* When full APIC register virtualization is implemented we'll have to make
14890	* sure APIC state is saved from the VMCS before IEM changes it.
14891	*/
14892	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14893	}
14894	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14895	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14896	else if (idMsr == MSR_K6_EFER)
14897	{
14898	/*
14899	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14900	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14901	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14902	*/
14903	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14904	}
14905
14906	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
14907	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14908	{
14909	switch (idMsr)
14910	{
14911	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14912	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14913	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14914	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14915	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14916	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14917	default:
14918	{
14919	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14920	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14921	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14922	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14923	break;
14924	}
14925	}
14926	}
14927	#ifdef VBOX_STRICT
14928	else
14929	{
14930	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14931	switch (idMsr)
14932	{
14933	case MSR_IA32_SYSENTER_CS:
14934	case MSR_IA32_SYSENTER_EIP:
14935	case MSR_IA32_SYSENTER_ESP:
14936	case MSR_K8_FS_BASE:
14937	case MSR_K8_GS_BASE:
14938	{
14939	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14940	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14941	}
14942
14943	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14944	default:
14945	{
14946	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14947	{
14948	/* EFER MSR writes are always intercepted. */
14949	if (idMsr != MSR_K6_EFER)
14950	{
14951	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14952	idMsr));
14953	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14954	}
14955	}
14956
14957	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14958	{
14959	Assert(pVmcsInfo->pvMsrBitmap);
14960	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14961	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14962	{
14963	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14964	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14965	}
14966	}
14967	break;
14968	}
14969	}
14970	}
14971	#endif /* VBOX_STRICT */
14972	}
14973	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14974	{
14975	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14976	rcStrict = VINF_SUCCESS;
14977	}
14978	else
14979	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14980
14981	return rcStrict;
14982	}
14983
14984
14985	/**
14986	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14987	*/
14988	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14989	{
14990	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14991
14992	/** @todo The guest has likely hit a contended spinlock. We might want to
14993	* poke a schedule different guest VCPU. */
14994	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14995	if (RT_SUCCESS(rc))
14996	return VINF_EM_RAW_INTERRUPT;
14997
14998	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14999	return rc;
15000	}
15001
15002
15003	/**
15004	* VM-exit handler for when the TPR value is lowered below the specified
15005	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15006	*/
15007	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15008	{
15009	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15010	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15011
15012	/*
15013	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15014	* We'll re-evaluate pending interrupts and inject them before the next VM
15015	* entry so we can just continue execution here.
15016	*/
15017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15018	return VINF_SUCCESS;
15019	}
15020
15021
15022	/**
15023	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15024	* VM-exit.
15025	*
15026	* @retval VINF_SUCCESS when guest execution can continue.
15027	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15028	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15029	* incompatible guest state for VMX execution (real-on-v86 case).
15030	*/
15031	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15032	{
15033	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15034	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15035
15036	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15037	hmR0VmxReadExitQualVmcs(pVmxTransient);
15038	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15039
15040	VBOXSTRICTRC rcStrict;
15041	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15042	uint64_t const uExitQual = pVmxTransient->uExitQual;
15043	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15044	switch (uAccessType)
15045	{
15046	/*
15047	* MOV to CRx.
15048	*/
15049	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15050	{
15051	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15052	AssertRCReturn(rc, rc);
15053
15054	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15055	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15056	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15057	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15058
15059	/*
15060	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15061	* - When nested paging isn't used.
15062	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15063	* - We are executing in the VM debug loop.
15064	*/
15065	Assert( iCrReg != 3
15066	\|\| !pVM->hm.s.fNestedPaging
15067	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15068	\|\| pVCpu->hm.s.fUsingDebugLoop);
15069
15070	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15071	Assert( iCrReg != 8
15072	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15073
15074	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15075	AssertMsg( rcStrict == VINF_SUCCESS
15076	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15077
15078	/*
15079	* This is a kludge for handling switches back to real mode when we try to use
15080	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15081	* deal with special selector values, so we have to return to ring-3 and run
15082	* there till the selector values are V86 mode compatible.
15083	*
15084	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15085	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15086	* this function.
15087	*/
15088	if ( iCrReg == 0
15089	&& rcStrict == VINF_SUCCESS
15090	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15091	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15092	&& (uOldCr0 & X86_CR0_PE)
15093	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15094	{
15095	/** @todo Check selectors rather than returning all the time. */
15096	Assert(!pVmxTransient->fIsNestedGuest);
15097	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15098	rcStrict = VINF_EM_RESCHEDULE_REM;
15099	}
15100	break;
15101	}
15102
15103	/*
15104	* MOV from CRx.
15105	*/
15106	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15107	{
15108	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15109	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15110
15111	/*
15112	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15113	* - When nested paging isn't used.
15114	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15115	* - We are executing in the VM debug loop.
15116	*/
15117	Assert( iCrReg != 3
15118	\|\| !pVM->hm.s.fNestedPaging
15119	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15120	\|\| pVCpu->hm.s.fUsingDebugLoop);
15121
15122	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15123	Assert( iCrReg != 8
15124	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15125
15126	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15127	break;
15128	}
15129
15130	/*
15131	* CLTS (Clear Task-Switch Flag in CR0).
15132	*/
15133	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15134	{
15135	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
15136	break;
15137	}
15138
15139	/*
15140	* LMSW (Load Machine-Status Word into CR0).
15141	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15142	*/
15143	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15144	{
15145	RTGCPTR GCPtrEffDst;
15146	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
15147	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15148	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15149	if (fMemOperand)
15150	{
15151	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
15152	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15153	}
15154	else
15155	GCPtrEffDst = NIL_RTGCPTR;
15156	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15157	break;
15158	}
15159
15160	default:
15161	{
15162	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15163	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15164	}
15165	}
15166
15167	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15168	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15169	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15170
15171	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15172	NOREF(pVM);
15173	return rcStrict;
15174	}
15175
15176
15177	/**
15178	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15179	* VM-exit.
15180	*/
15181	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15182	{
15183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15184	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15185
15186	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15187	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15188	hmR0VmxReadExitQualVmcs(pVmxTransient);
15189	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15190	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15191	\| CPUMCTX_EXTRN_EFER);
15192	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15193	AssertRCReturn(rc, rc);
15194
15195	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15196	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15197	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15198	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15199	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15200	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15201	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15202	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15203
15204	/*
15205	* Update exit history to see if this exit can be optimized.
15206	*/
15207	VBOXSTRICTRC rcStrict;
15208	PCEMEXITREC pExitRec = NULL;
15209	if ( !fGstStepping
15210	&& !fDbgStepping)
15211	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15212	!fIOString
15213	? !fIOWrite
15214	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15215	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15216	: !fIOWrite
15217	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15218	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15219	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15220	if (!pExitRec)
15221	{
15222	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15223	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15224
15225	uint32_t const cbValue = s_aIOSizes[uIOSize];
15226	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
15227	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15228	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15229	if (fIOString)
15230	{
15231	/*
15232	* INS/OUTS - I/O String instruction.
15233	*
15234	* Use instruction-information if available, otherwise fall back on
15235	* interpreting the instruction.
15236	*/
15237	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15238	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15239	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15240	if (fInsOutsInfo)
15241	{
15242	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15243	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15244	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15245	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15246	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15247	if (fIOWrite)
15248	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15249	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15250	else
15251	{
15252	/*
15253	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15254	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15255	* See Intel Instruction spec. for "INS".
15256	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15257	*/
15258	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15259	}
15260	}
15261	else
15262	rcStrict = IEMExecOne(pVCpu);
15263
15264	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15265	fUpdateRipAlready = true;
15266	}
15267	else
15268	{
15269	/*
15270	* IN/OUT - I/O instruction.
15271	*/
15272	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15273	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15274	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15275	if (fIOWrite)
15276	{
15277	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15278	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15279	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15280	&& !pCtx->eflags.Bits.u1TF)
15281	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15282	}
15283	else
15284	{
15285	uint32_t u32Result = 0;
15286	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15287	if (IOM_SUCCESS(rcStrict))
15288	{
15289	/* Save result of I/O IN instr. in AL/AX/EAX. */
15290	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15291	}
15292	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15293	&& !pCtx->eflags.Bits.u1TF)
15294	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15295	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15296	}
15297	}
15298
15299	if (IOM_SUCCESS(rcStrict))
15300	{
15301	if (!fUpdateRipAlready)
15302	{
15303	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15304	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15305	}
15306
15307	/*
15308	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15309	* while booting Fedora 17 64-bit guest.
15310	*
15311	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15312	*/
15313	if (fIOString)
15314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15315
15316	/*
15317	* If any I/O breakpoints are armed, we need to check if one triggered
15318	* and take appropriate action.
15319	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15320	*/
15321	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15322	AssertRCReturn(rc, rc);
15323
15324	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15325	* execution engines about whether hyper BPs and such are pending. */
15326	uint32_t const uDr7 = pCtx->dr[7];
15327	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15328	&& X86_DR7_ANY_RW_IO(uDr7)
15329	&& (pCtx->cr4 & X86_CR4_DE))
15330	\|\| DBGFBpIsHwIoArmed(pVM)))
15331	{
15332	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15333
15334	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15335	VMMRZCallRing3Disable(pVCpu);
15336	HM_DISABLE_PREEMPT(pVCpu);
15337
15338	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15339
15340	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15341	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15342	{
15343	/* Raise #DB. */
15344	if (fIsGuestDbgActive)
15345	ASMSetDR6(pCtx->dr[6]);
15346	if (pCtx->dr[7] != uDr7)
15347	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15348
15349	hmR0VmxSetPendingXcptDB(pVCpu);
15350	}
15351	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15352	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15353	else if ( rcStrict2 != VINF_SUCCESS
15354	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15355	rcStrict = rcStrict2;
15356	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15357
15358	HM_RESTORE_PREEMPT();
15359	VMMRZCallRing3Enable(pVCpu);
15360	}
15361	}
15362
15363	#ifdef VBOX_STRICT
15364	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15365	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15366	Assert(!fIOWrite);
15367	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15368	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15369	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15370	Assert(fIOWrite);
15371	else
15372	{
15373	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15374	* statuses, that the VMM device and some others may return. See
15375	* IOM_SUCCESS() for guidance. */
15376	AssertMsg( RT_FAILURE(rcStrict)
15377	\|\| rcStrict == VINF_SUCCESS
15378	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15379	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15380	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15381	\|\| rcStrict == VINF_EM_RAW_TO_R3
15382	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15383	# endif
15384	}
15385	#endif
15386	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15387	}
15388	else
15389	{
15390	/*
15391	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15392	*/
15393	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15394	AssertRCReturn(rc2, rc2);
15395	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15396	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15397	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15398	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15399	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15400	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15401
15402	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15403	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15404
15405	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15406	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15407	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15408	}
15409	return rcStrict;
15410	}
15411
15412
15413	/**
15414	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15415	* VM-exit.
15416	*/
15417	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15418	{
15419	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15420
15421	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15422	hmR0VmxReadExitQualVmcs(pVmxTransient);
15423	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15424	{
15425	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15426	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15427	{
15428	uint32_t uErrCode;
15429	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15430	{
15431	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15432	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15433	}
15434	else
15435	uErrCode = 0;
15436
15437	RTGCUINTPTR GCPtrFaultAddress;
15438	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15439	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15440	else
15441	GCPtrFaultAddress = 0;
15442
15443	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15444
15445	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15446	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
15447
15448	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15449	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15450	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15451	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15452	}
15453	}
15454
15455	/* Fall back to the interpreter to emulate the task-switch. */
15456	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15457	return VERR_EM_INTERPRETER;
15458	}
15459
15460
15461	/**
15462	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15463	*/
15464	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15465	{
15466	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15467
15468	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15469	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15470	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15471	AssertRC(rc);
15472	return VINF_EM_DBG_STEPPED;
15473	}
15474
15475
15476	/**
15477	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15478	*/
15479	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15480	{
15481	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15482	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15483
15484	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15485	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15486	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15487	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15488	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15489
15490	/*
15491	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15492	*/
15493	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15494	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15495	{
15496	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15497	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15498	{
15499	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15500	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15501	}
15502	}
15503	else
15504	{
15505	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15506	return rcStrict;
15507	}
15508
15509	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15510	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15511	hmR0VmxReadExitQualVmcs(pVmxTransient);
15512	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15513	AssertRCReturn(rc, rc);
15514
15515	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15516	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15517	switch (uAccessType)
15518	{
15519	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15520	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15521	{
15522	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15523	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15524	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15525
15526	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15527	GCPhys &= PAGE_BASE_GC_MASK;
15528	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15529	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15530	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15531
15532	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15533	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15534	rcStrict = IOMMMIOPhysHandler(pVM, pVCpu,
15535	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15536	CPUMCTX2CORE(pCtx), GCPhys);
15537	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15538	if ( rcStrict == VINF_SUCCESS
15539	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15540	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15541	{
15542	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15543	\| HM_CHANGED_GUEST_APIC_TPR);
15544	rcStrict = VINF_SUCCESS;
15545	}
15546	break;
15547	}
15548
15549	default:
15550	{
15551	Log4Func(("uAccessType=%#x\n", uAccessType));
15552	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15553	break;
15554	}
15555	}
15556
15557	if (rcStrict != VINF_SUCCESS)
15558	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15559	return rcStrict;
15560	}
15561
15562
15563	/**
15564	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15565	* VM-exit.
15566	*/
15567	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15568	{
15569	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15570	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15571
15572	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15573	if (!pVmxTransient->fIsNestedGuest)
15574	{
15575	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15576	if (pVmxTransient->fWasGuestDebugStateActive)
15577	{
15578	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15579	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15580	}
15581
15582	if ( !pVCpu->hm.s.fSingleInstruction
15583	&& !pVmxTransient->fWasHyperDebugStateActive)
15584	{
15585	Assert(!DBGFIsStepping(pVCpu));
15586	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15587
15588	/* Don't intercept MOV DRx any more. */
15589	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15590	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15591	AssertRC(rc);
15592
15593	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15594	VMMRZCallRing3Disable(pVCpu);
15595	HM_DISABLE_PREEMPT(pVCpu);
15596
15597	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15598	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15599	Assert(CPUMIsGuestDebugStateActive(pVCpu));
15600
15601	HM_RESTORE_PREEMPT();
15602	VMMRZCallRing3Enable(pVCpu);
15603
15604	#ifdef VBOX_WITH_STATISTICS
15605	hmR0VmxReadExitQualVmcs(pVmxTransient);
15606	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15607	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15608	else
15609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15610	#endif
15611	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15612	return VINF_SUCCESS;
15613	}
15614	}
15615
15616	/*
15617	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15618	* The EFER MSR is always up-to-date.
15619	* Update the segment registers and DR7 from the CPU.
15620	*/
15621	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15622	hmR0VmxReadExitQualVmcs(pVmxTransient);
15623	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15624	AssertRCReturn(rc, rc);
15625	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15626
15627	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15628	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15629	{
15630	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15631	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15632	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15633	if (RT_SUCCESS(rc))
15634	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15635	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15636	}
15637	else
15638	{
15639	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15640	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15641	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15642	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15643	}
15644
15645	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15646	if (RT_SUCCESS(rc))
15647	{
15648	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15649	AssertRCReturn(rc2, rc2);
15650	return VINF_SUCCESS;
15651	}
15652	return rc;
15653	}
15654
15655
15656	/**
15657	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15658	* Conditional VM-exit.
15659	*/
15660	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15661	{
15662	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15663	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15664
15665	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15666	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15667	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15668	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15669	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15670
15671	/*
15672	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15673	*/
15674	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15675	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15676	{
15677	/*
15678	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15679	* instruction emulation to inject the original event. Otherwise, injecting the original event
15680	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
15681	*/
15682	if (!pVCpu->hm.s.Event.fPending)
15683	{ /* likely */ }
15684	else
15685	{
15686	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15687	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15688	/** @todo NSTVMX: Think about how this should be handled. */
15689	if (pVmxTransient->fIsNestedGuest)
15690	return VERR_VMX_IPE_3;
15691	#endif
15692	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15693	}
15694	}
15695	else
15696	{
15697	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15698	return rcStrict;
15699	}
15700
15701	/*
15702	* Get sufficient state and update the exit history entry.
15703	*/
15704	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15705	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15706	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15707	AssertRCReturn(rc, rc);
15708
15709	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15710	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15711	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15712	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15713	if (!pExitRec)
15714	{
15715	/*
15716	* If we succeed, resume guest execution.
15717	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15718	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15719	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15720	* weird case. See @bugref{6043}.
15721	*/
15722	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15723	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15724	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15725	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15726	if ( rcStrict == VINF_SUCCESS
15727	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15728	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15729	{
15730	/* Successfully handled MMIO operation. */
15731	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15732	\| HM_CHANGED_GUEST_APIC_TPR);
15733	rcStrict = VINF_SUCCESS;
15734	}
15735	}
15736	else
15737	{
15738	/*
15739	* Frequent exit or something needing probing. Call EMHistoryExec.
15740	*/
15741	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15742	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15743
15744	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15745	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15746
15747	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15748	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15749	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15750	}
15751	return rcStrict;
15752	}
15753
15754
15755	/**
15756	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15757	* VM-exit.
15758	*/
15759	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15760	{
15761	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15762	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15763
15764	hmR0VmxReadExitQualVmcs(pVmxTransient);
15765	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15766	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15767	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15768	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15769	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15770
15771	/*
15772	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15773	*/
15774	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15775	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15776	{
15777	/*
15778	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
15779	* we shall resolve the nested #PF and re-inject the original event.
15780	*/
15781	if (pVCpu->hm.s.Event.fPending)
15782	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
15783	}
15784	else
15785	{
15786	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15787	return rcStrict;
15788	}
15789
15790	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15791	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15792	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15793	AssertRCReturn(rc, rc);
15794
15795	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15796	uint64_t const uExitQual = pVmxTransient->uExitQual;
15797	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
15798
15799	RTGCUINT uErrorCode = 0;
15800	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15801	uErrorCode \|= X86_TRAP_PF_ID;
15802	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15803	uErrorCode \|= X86_TRAP_PF_RW;
15804	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15805	uErrorCode \|= X86_TRAP_PF_P;
15806
15807	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15808	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15809	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
15810
15811	/*
15812	* Handle the pagefault trap for the nested shadow table.
15813	*/
15814	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15815	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15816	TRPMResetTrap(pVCpu);
15817
15818	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15819	if ( rcStrict == VINF_SUCCESS
15820	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15821	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15822	{
15823	/* Successfully synced our nested page tables. */
15824	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15825	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15826	return VINF_SUCCESS;
15827	}
15828
15829	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15830	return rcStrict;
15831	}
15832
15833
15834	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15835	/**
15836	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15837	*/
15838	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15839	{
15840	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15841
15842	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15843	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15844	hmR0VmxReadExitQualVmcs(pVmxTransient);
15845	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15846	\| CPUMCTX_EXTRN_HWVIRT
15847	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15848	AssertRCReturn(rc, rc);
15849
15850	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15851
15852	VMXVEXITINFO ExitInfo;
15853	RT_ZERO(ExitInfo);
15854	ExitInfo.uReason = pVmxTransient->uExitReason;
15855	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15856	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15857	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15858	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15859
15860	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15861	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15862	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15863	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15864	{
15865	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15866	rcStrict = VINF_SUCCESS;
15867	}
15868	return rcStrict;
15869	}
15870
15871
15872	/**
15873	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15874	*/
15875	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15876	{
15877	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15878
15879	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15880	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15881	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15882	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15883	AssertRCReturn(rc, rc);
15884
15885	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15886
15887	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
15888	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
15889	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
15890	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15891	{
15892	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15893	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
15894	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15895	}
15896	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15897	return rcStrict;
15898	}
15899
15900
15901	/**
15902	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15903	*/
15904	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15905	{
15906	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15907
15908	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15909	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15910	hmR0VmxReadExitQualVmcs(pVmxTransient);
15911	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15912	\| CPUMCTX_EXTRN_HWVIRT
15913	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15914	AssertRCReturn(rc, rc);
15915
15916	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15917
15918	VMXVEXITINFO ExitInfo;
15919	RT_ZERO(ExitInfo);
15920	ExitInfo.uReason = pVmxTransient->uExitReason;
15921	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15922	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15923	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15924	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15925
15926	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15927	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15928	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15929	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15930	{
15931	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15932	rcStrict = VINF_SUCCESS;
15933	}
15934	return rcStrict;
15935	}
15936
15937
15938	/**
15939	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15940	*/
15941	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15942	{
15943	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15944
15945	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15946	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15947	hmR0VmxReadExitQualVmcs(pVmxTransient);
15948	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15949	\| CPUMCTX_EXTRN_HWVIRT
15950	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15951	AssertRCReturn(rc, rc);
15952
15953	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15954
15955	VMXVEXITINFO ExitInfo;
15956	RT_ZERO(ExitInfo);
15957	ExitInfo.uReason = pVmxTransient->uExitReason;
15958	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15959	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15960	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15961	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15962
15963	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15964	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15965	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15966	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15967	{
15968	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15969	rcStrict = VINF_SUCCESS;
15970	}
15971	return rcStrict;
15972	}
15973
15974
15975	/**
15976	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
15977	*/
15978	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15979	{
15980	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15981
15982	/*
15983	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
15984	* thus might not need to import the shadow VMCS state, it's safer just in case
15985	* code elsewhere dares look at unsynced VMCS fields.
15986	*/
15987	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15988	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15989	hmR0VmxReadExitQualVmcs(pVmxTransient);
15990	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15991	\| CPUMCTX_EXTRN_HWVIRT
15992	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15993	AssertRCReturn(rc, rc);
15994
15995	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15996
15997	VMXVEXITINFO ExitInfo;
15998	RT_ZERO(ExitInfo);
15999	ExitInfo.uReason = pVmxTransient->uExitReason;
16000	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16001	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16002	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16003	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16004	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16005
16006	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16007	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16008	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16009	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16010	{
16011	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16012	rcStrict = VINF_SUCCESS;
16013	}
16014	return rcStrict;
16015	}
16016
16017
16018	/**
16019	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16020	*/
16021	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16022	{
16023	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16024
16025	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16026	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16027	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16028	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16029	AssertRCReturn(rc, rc);
16030
16031	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16032
16033	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16034	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
16035	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16036	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16037	{
16038	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16039	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16040	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16041	}
16042	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16043	return rcStrict;
16044	}
16045
16046
16047	/**
16048	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16049	*/
16050	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16051	{
16052	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16053
16054	/*
16055	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16056	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16057	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16058	*/
16059	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16060	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16061	hmR0VmxReadExitQualVmcs(pVmxTransient);
16062	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16063	\| CPUMCTX_EXTRN_HWVIRT
16064	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16065	AssertRCReturn(rc, rc);
16066
16067	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16068
16069	VMXVEXITINFO ExitInfo;
16070	RT_ZERO(ExitInfo);
16071	ExitInfo.uReason = pVmxTransient->uExitReason;
16072	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16073	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16074	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16075	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16076	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16077
16078	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16079	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16080	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16081	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16082	{
16083	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16084	rcStrict = VINF_SUCCESS;
16085	}
16086	return rcStrict;
16087	}
16088
16089
16090	/**
16091	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16092	*/
16093	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16094	{
16095	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16096
16097	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16098	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16099	\| CPUMCTX_EXTRN_HWVIRT
16100	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16101	AssertRCReturn(rc, rc);
16102
16103	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16104
16105	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
16106	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16107	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16108	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16109	{
16110	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16111	rcStrict = VINF_SUCCESS;
16112	}
16113	return rcStrict;
16114	}
16115
16116
16117	/**
16118	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16119	*/
16120	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16121	{
16122	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16123
16124	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16125	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16126	hmR0VmxReadExitQualVmcs(pVmxTransient);
16127	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16128	\| CPUMCTX_EXTRN_HWVIRT
16129	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16130	AssertRCReturn(rc, rc);
16131
16132	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16133
16134	VMXVEXITINFO ExitInfo;
16135	RT_ZERO(ExitInfo);
16136	ExitInfo.uReason = pVmxTransient->uExitReason;
16137	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16138	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16139	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16140	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16141
16142	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16143	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16144	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16145	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16146	{
16147	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16148	rcStrict = VINF_SUCCESS;
16149	}
16150	return rcStrict;
16151	}
16152
16153
16154	/**
16155	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16156	*/
16157	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16158	{
16159	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16160
16161	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16162	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16163	hmR0VmxReadExitQualVmcs(pVmxTransient);
16164	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16165	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16166	AssertRCReturn(rc, rc);
16167
16168	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16169
16170	VMXVEXITINFO ExitInfo;
16171	RT_ZERO(ExitInfo);
16172	ExitInfo.uReason = pVmxTransient->uExitReason;
16173	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16174	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16175	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16176	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16177
16178	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16179	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16180	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16181	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16182	{
16183	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16184	rcStrict = VINF_SUCCESS;
16185	}
16186	return rcStrict;
16187	}
16188	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16189	/** @} */
16190
16191
16192	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16193	/** @name Nested-guest VM-exit handlers.
16194	* @{
16195	*/
16196	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16197	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16198	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16199
16200	/**
16201	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16202	* Conditional VM-exit.
16203	*/
16204	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16205	{
16206	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16207
16208	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16209
16210	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16211	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16212	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16213
16214	switch (uExitIntType)
16215	{
16216	/*
16217	* Physical NMIs:
16218	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16219	*/
16220	case VMX_EXIT_INT_INFO_TYPE_NMI:
16221	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16222
16223	/*
16224	* Hardware exceptions,
16225	* Software exceptions,
16226	* Privileged software exceptions:
16227	* Figure out if the exception must be delivered to the guest or the nested-guest.
16228	*/
16229	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16230	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16231	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16232	{
16233	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16234	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16235	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16236	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16237
16238	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16239	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16240	pVmxTransient->uExitIntErrorCode);
16241	if (fIntercept)
16242	{
16243	/* Exit qualification is required for debug and page-fault exceptions. */
16244	hmR0VmxReadExitQualVmcs(pVmxTransient);
16245
16246	/*
16247	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16248	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16249	* length. However, if delivery of a software interrupt, software exception or privileged
16250	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16251	*/
16252	VMXVEXITINFO ExitInfo;
16253	RT_ZERO(ExitInfo);
16254	ExitInfo.uReason = pVmxTransient->uExitReason;
16255	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16256	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16257
16258	VMXVEXITEVENTINFO ExitEventInfo;
16259	RT_ZERO(ExitEventInfo);
16260	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16261	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16262	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16263	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16264
16265	#ifdef DEBUG_ramshankar
16266	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16267	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
16268	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16269	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16270	{
16271	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
16272	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
16273	}
16274	#endif
16275	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16276	}
16277
16278	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16279	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16280	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16281	}
16282
16283	/*
16284	* Software interrupts:
16285	* VM-exits cannot be caused by software interrupts.
16286	*
16287	* External interrupts:
16288	* This should only happen when "acknowledge external interrupts on VM-exit"
16289	* control is set. However, we never set this when executing a guest or
16290	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16291	* the guest.
16292	*/
16293	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16294	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16295	default:
16296	{
16297	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16298	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16299	}
16300	}
16301	}
16302
16303
16304	/**
16305	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16306	* Unconditional VM-exit.
16307	*/
16308	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16309	{
16310	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16311	return IEMExecVmxVmexitTripleFault(pVCpu);
16312	}
16313
16314
16315	/**
16316	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16317	*/
16318	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16319	{
16320	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16321
16322	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16323	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16324	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16325	}
16326
16327
16328	/**
16329	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16330	*/
16331	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16332	{
16333	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16334
16335	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16336	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16337	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16338	}
16339
16340
16341	/**
16342	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16343	* Unconditional VM-exit.
16344	*/
16345	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16346	{
16347	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16348
16349	hmR0VmxReadExitQualVmcs(pVmxTransient);
16350	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16351	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16352	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16353
16354	VMXVEXITINFO ExitInfo;
16355	RT_ZERO(ExitInfo);
16356	ExitInfo.uReason = pVmxTransient->uExitReason;
16357	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16358	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16359
16360	VMXVEXITEVENTINFO ExitEventInfo;
16361	RT_ZERO(ExitEventInfo);
16362	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16363	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16364	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16365	}
16366
16367
16368	/**
16369	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16370	*/
16371	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16372	{
16373	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16374
16375	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16376	{
16377	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16378	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16379	}
16380	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16381	}
16382
16383
16384	/**
16385	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16386	*/
16387	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16388	{
16389	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16390
16391	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16392	{
16393	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16394	hmR0VmxReadExitQualVmcs(pVmxTransient);
16395
16396	VMXVEXITINFO ExitInfo;
16397	RT_ZERO(ExitInfo);
16398	ExitInfo.uReason = pVmxTransient->uExitReason;
16399	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16400	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16401	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16402	}
16403	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16404	}
16405
16406
16407	/**
16408	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16409	*/
16410	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16411	{
16412	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16413
16414	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16415	{
16416	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16417	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16418	}
16419	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16420	}
16421
16422
16423	/**
16424	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16425	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16426	*/
16427	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16428	{
16429	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16430
16431	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16432	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16433
16434	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16435
16436	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16437	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16438	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16439
16440	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16441	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16442	u64VmcsField &= UINT64_C(0xffffffff);
16443
16444	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16445	{
16446	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16447	hmR0VmxReadExitQualVmcs(pVmxTransient);
16448
16449	VMXVEXITINFO ExitInfo;
16450	RT_ZERO(ExitInfo);
16451	ExitInfo.uReason = pVmxTransient->uExitReason;
16452	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16453	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16454	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16455	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16456	}
16457
16458	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16459	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16460	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16461	}
16462
16463
16464	/**
16465	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16466	*/
16467	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16468	{
16469	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16470
16471	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16472	{
16473	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16474	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16475	}
16476
16477	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16478	}
16479
16480
16481	/**
16482	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16483	* Conditional VM-exit.
16484	*/
16485	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16486	{
16487	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16488
16489	hmR0VmxReadExitQualVmcs(pVmxTransient);
16490	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16491
16492	VBOXSTRICTRC rcStrict;
16493	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16494	switch (uAccessType)
16495	{
16496	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16497	{
16498	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16499	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16500	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16501	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16502
16503	bool fIntercept;
16504	switch (iCrReg)
16505	{
16506	case 0:
16507	case 4:
16508	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16509	break;
16510
16511	case 3:
16512	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16513	break;
16514
16515	case 8:
16516	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16517	break;
16518
16519	default:
16520	fIntercept = false;
16521	break;
16522	}
16523	if (fIntercept)
16524	{
16525	VMXVEXITINFO ExitInfo;
16526	RT_ZERO(ExitInfo);
16527	ExitInfo.uReason = pVmxTransient->uExitReason;
16528	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16529	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16530	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16531	}
16532	else
16533	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16534	break;
16535	}
16536
16537	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16538	{
16539	/*
16540	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16541	* CR2 reads do not cause a VM-exit.
16542	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16543	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16544	*/
16545	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16546	if ( iCrReg == 3
16547	\|\| iCrReg == 8)
16548	{
16549	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16550	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16551	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16552	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16553	{
16554	VMXVEXITINFO ExitInfo;
16555	RT_ZERO(ExitInfo);
16556	ExitInfo.uReason = pVmxTransient->uExitReason;
16557	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16558	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16559	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16560	}
16561	else
16562	{
16563	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16564	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16565	}
16566	}
16567	else
16568	{
16569	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16570	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16571	}
16572	break;
16573	}
16574
16575	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16576	{
16577	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16578	Assert(pVmcsNstGst);
16579	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16580	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16581	if ( (uGstHostMask & X86_CR0_TS)
16582	&& (uReadShadow & X86_CR0_TS))
16583	{
16584	VMXVEXITINFO ExitInfo;
16585	RT_ZERO(ExitInfo);
16586	ExitInfo.uReason = pVmxTransient->uExitReason;
16587	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16588	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16589	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16590	}
16591	else
16592	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
16593	break;
16594	}
16595
16596	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16597	{
16598	RTGCPTR GCPtrEffDst;
16599	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16600	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16601	if (fMemOperand)
16602	{
16603	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16604	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16605	}
16606	else
16607	GCPtrEffDst = NIL_RTGCPTR;
16608
16609	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16610	{
16611	VMXVEXITINFO ExitInfo;
16612	RT_ZERO(ExitInfo);
16613	ExitInfo.uReason = pVmxTransient->uExitReason;
16614	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16615	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16616	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16617	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16618	}
16619	else
16620	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
16621	break;
16622	}
16623
16624	default:
16625	{
16626	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16627	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16628	}
16629	}
16630
16631	if (rcStrict == VINF_IEM_RAISED_XCPT)
16632	{
16633	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16634	rcStrict = VINF_SUCCESS;
16635	}
16636	return rcStrict;
16637	}
16638
16639
16640	/**
16641	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16642	* Conditional VM-exit.
16643	*/
16644	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16645	{
16646	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16647
16648	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16649	{
16650	hmR0VmxReadExitQualVmcs(pVmxTransient);
16651	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16652
16653	VMXVEXITINFO ExitInfo;
16654	RT_ZERO(ExitInfo);
16655	ExitInfo.uReason = pVmxTransient->uExitReason;
16656	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16657	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16658	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16659	}
16660	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16661	}
16662
16663
16664	/**
16665	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16666	* Conditional VM-exit.
16667	*/
16668	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16669	{
16670	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16671
16672	hmR0VmxReadExitQualVmcs(pVmxTransient);
16673
16674	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16675	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16676	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16677
16678	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16679	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16680	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16681	{
16682	/*
16683	* IN/OUT instruction:
16684	* - Provides VM-exit instruction length.
16685	*
16686	* INS/OUTS instruction:
16687	* - Provides VM-exit instruction length.
16688	* - Provides Guest-linear address.
16689	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16690	*/
16691	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16692	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16693
16694	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16695	pVmxTransient->ExitInstrInfo.u = 0;
16696	pVmxTransient->uGuestLinearAddr = 0;
16697
16698	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16699	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16700	if (fIOString)
16701	{
16702	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16703	if (fVmxInsOutsInfo)
16704	{
16705	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16706	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16707	}
16708	}
16709
16710	VMXVEXITINFO ExitInfo;
16711	RT_ZERO(ExitInfo);
16712	ExitInfo.uReason = pVmxTransient->uExitReason;
16713	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16714	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16715	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16716	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16717	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16718	}
16719	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16720	}
16721
16722
16723	/**
16724	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16725	*/
16726	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16727	{
16728	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16729
16730	uint32_t fMsrpm;
16731	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16732	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16733	else
16734	fMsrpm = VMXMSRPM_EXIT_RD;
16735
16736	if (fMsrpm & VMXMSRPM_EXIT_RD)
16737	{
16738	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16739	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16740	}
16741	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16742	}
16743
16744
16745	/**
16746	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16747	*/
16748	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16749	{
16750	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16751
16752	uint32_t fMsrpm;
16753	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16754	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16755	else
16756	fMsrpm = VMXMSRPM_EXIT_WR;
16757
16758	if (fMsrpm & VMXMSRPM_EXIT_WR)
16759	{
16760	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16761	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16762	}
16763	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16764	}
16765
16766
16767	/**
16768	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16769	*/
16770	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16771	{
16772	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16773
16774	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16775	{
16776	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16777	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16778	}
16779	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16780	}
16781
16782
16783	/**
16784	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16785	* VM-exit.
16786	*/
16787	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16788	{
16789	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16790
16791	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16792	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16793	VMXVEXITINFO ExitInfo;
16794	RT_ZERO(ExitInfo);
16795	ExitInfo.uReason = pVmxTransient->uExitReason;
16796	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16797	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16798	}
16799
16800
16801	/**
16802	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16803	*/
16804	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16805	{
16806	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16807
16808	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16809	{
16810	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16811	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16812	}
16813	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16814	}
16815
16816
16817	/**
16818	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16819	*/
16820	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16821	{
16822	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16823
16824	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16825	* PAUSE when executing a nested-guest? If it does not, we would not need
16826	* to check for the intercepts here. Just call VM-exit... */
16827
16828	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16829	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16830	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16831	{
16832	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16833	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16834	}
16835	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16836	}
16837
16838
16839	/**
16840	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16841	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16842	*/
16843	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16844	{
16845	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16846
16847	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16848	{
16849	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16850	VMXVEXITINFO ExitInfo;
16851	RT_ZERO(ExitInfo);
16852	ExitInfo.uReason = pVmxTransient->uExitReason;
16853	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16854	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16855	}
16856	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16857	}
16858
16859
16860	/**
16861	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16862	* VM-exit.
16863	*/
16864	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16865	{
16866	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16867
16868	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16869	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16870	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16871	hmR0VmxReadExitQualVmcs(pVmxTransient);
16872
16873	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16874
16875	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
16876	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
16877
16878	VMXVEXITINFO ExitInfo;
16879	RT_ZERO(ExitInfo);
16880	ExitInfo.uReason = pVmxTransient->uExitReason;
16881	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16882	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16883
16884	VMXVEXITEVENTINFO ExitEventInfo;
16885	RT_ZERO(ExitEventInfo);
16886	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16887	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16888	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
16889	}
16890
16891
16892	/**
16893	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
16894	* Conditional VM-exit.
16895	*/
16896	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16897	{
16898	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16899
16900	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
16901	hmR0VmxReadExitQualVmcs(pVmxTransient);
16902	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16903	}
16904
16905
16906	/**
16907	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
16908	* Conditional VM-exit.
16909	*/
16910	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16911	{
16912	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16913
16914	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
16915	hmR0VmxReadExitQualVmcs(pVmxTransient);
16916	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16917	}
16918
16919
16920	/**
16921	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16922	*/
16923	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16924	{
16925	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16926
16927	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16928	{
16929	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16930	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16931	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16932	}
16933	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16934	}
16935
16936
16937	/**
16938	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16939	*/
16940	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16941	{
16942	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16943
16944	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16945	{
16946	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16947	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16948	}
16949	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16950	}
16951
16952
16953	/**
16954	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16955	*/
16956	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16957	{
16958	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16959
16960	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16961	{
16962	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
16963	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16964	hmR0VmxReadExitQualVmcs(pVmxTransient);
16965	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16966
16967	VMXVEXITINFO ExitInfo;
16968	RT_ZERO(ExitInfo);
16969	ExitInfo.uReason = pVmxTransient->uExitReason;
16970	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16971	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16972	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16973	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16974	}
16975	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
16976	}
16977
16978
16979	/**
16980	* Nested-guest VM-exit handler for invalid-guest state
16981	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
16982	*/
16983	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16984	{
16985	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16986
16987	/*
16988	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
16989	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
16990	* Handle it like it's in an invalid guest state of the outer guest.
16991	*
16992	* When the fast path is implemented, this should be changed to cause the corresponding
16993	* nested-guest VM-exit.
16994	*/
16995	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
16996	}
16997
16998
16999	/**
17000	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17001	* and only provide the instruction length.
17002	*
17003	* Unconditional VM-exit.
17004	*/
17005	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17006	{
17007	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17008
17009	#ifdef VBOX_STRICT
17010	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17011	switch (pVmxTransient->uExitReason)
17012	{
17013	case VMX_EXIT_ENCLS:
17014	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17015	break;
17016
17017	case VMX_EXIT_VMFUNC:
17018	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17019	break;
17020	}
17021	#endif
17022
17023	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17024	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17025	}
17026
17027
17028	/**
17029	* Nested-guest VM-exit handler for instructions that provide instruction length as
17030	* well as more information.
17031	*
17032	* Unconditional VM-exit.
17033	*/
17034	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17035	{
17036	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17037
17038	#ifdef VBOX_STRICT
17039	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17040	switch (pVmxTransient->uExitReason)
17041	{
17042	case VMX_EXIT_GDTR_IDTR_ACCESS:
17043	case VMX_EXIT_LDTR_TR_ACCESS:
17044	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17045	break;
17046
17047	case VMX_EXIT_RDRAND:
17048	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17049	break;
17050
17051	case VMX_EXIT_RDSEED:
17052	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17053	break;
17054
17055	case VMX_EXIT_XSAVES:
17056	case VMX_EXIT_XRSTORS:
17057	/** @todo NSTVMX: Verify XSS-bitmap. */
17058	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17059	break;
17060
17061	case VMX_EXIT_UMWAIT:
17062	case VMX_EXIT_TPAUSE:
17063	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17064	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17065	break;
17066	}
17067	#endif
17068
17069	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17070	hmR0VmxReadExitQualVmcs(pVmxTransient);
17071	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17072
17073	VMXVEXITINFO ExitInfo;
17074	RT_ZERO(ExitInfo);
17075	ExitInfo.uReason = pVmxTransient->uExitReason;
17076	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17077	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17078	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17079	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17080	}
17081
17082	/** @} */
17083	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17084

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 81207

Download in other formats: