HMVMXR0.cpp@ 87372

Last change on this file since 87372 was 87345, checked in by vboxsync, 4 years ago
VMM/CPUM: Removed CPUM_SYNC_FPU_STATE, CPUM_SYNC_DEBUG_REGS_GUEST and CPUM_SYNC_DEBUG_REGS_HYPER as they were related to 64-bit guest on 32-bit host support and is no longer used.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 722.6 KB

Line
1	/* $Id: HMVMXR0.cpp 87345 2021-01-21 11:37:30Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#include "HMInternal.h"
39	#include <VBox/vmm/vmcc.h>
40	#include <VBox/vmm/hmvmxinline.h>
41	#include "HMVMXR0.h"
42	#include "dtrace/VBoxVMM.h"
43
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
47	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/**
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
83	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
84
85	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
86	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
87	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
88	\| HMVMX_READ_EXIT_INSTR_LEN \
89	\| HMVMX_READ_IDT_VECTORING_INFO \
90	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
91
92	/** Assert that all the given fields have been read from the VMCS. */
93	#ifdef VBOX_STRICT
94	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
95	do { \
96	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
97	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
98	} while (0)
99	#else
100	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
101	#endif
102
103	/**
104	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
105	* guest using hardware-assisted VMX.
106	*
107	* This excludes state like GPRs (other than RSP) which are always are
108	* swapped and restored across the world-switch and also registers like EFER,
109	* MSR which cannot be modified by the guest without causing a VM-exit.
110	*/
111	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
112	\| CPUMCTX_EXTRN_RFLAGS \
113	\| CPUMCTX_EXTRN_RSP \
114	\| CPUMCTX_EXTRN_SREG_MASK \
115	\| CPUMCTX_EXTRN_TABLE_MASK \
116	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
117	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
118	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
119	\| CPUMCTX_EXTRN_TSC_AUX \
120	\| CPUMCTX_EXTRN_OTHER_MSRS \
121	\| CPUMCTX_EXTRN_CR0 \
122	\| CPUMCTX_EXTRN_CR3 \
123	\| CPUMCTX_EXTRN_CR4 \
124	\| CPUMCTX_EXTRN_DR7 \
125	\| CPUMCTX_EXTRN_HWVIRT \
126	\| CPUMCTX_EXTRN_HM_VMX_MASK)
127
128	/**
129	* Exception bitmap mask for real-mode guests (real-on-v86).
130	*
131	* We need to intercept all exceptions manually except:
132	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
133	* due to bugs in Intel CPUs.
134	* - \#PF need not be intercepted even in real-mode if we have nested paging
135	* support.
136	*/
137	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
138	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
139	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
140	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
141	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
142	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
143	\| RT_BIT(X86_XCPT_XF))
144
145	/** Maximum VM-instruction error number. */
146	#define HMVMX_INSTR_ERROR_MAX 28
147
148	/** Profiling macro. */
149	#ifdef HM_PROFILE_EXIT_DISPATCH
150	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
151	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
152	#else
153	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
154	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
155	#endif
156
157	/** Assert that preemption is disabled or covered by thread-context hooks. */
158	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
159	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
160
161	/** Assert that we haven't migrated CPUs when thread-context hooks are not
162	* used. */
163	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
164	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
165	("Illegal migration! Entered on CPU %u Current %u\n", \
166	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
167
168	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
169	* context. */
170	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
171	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
172	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
173
174	/** Log the VM-exit reason with an easily visible marker to identify it in a
175	* potential sea of logging data. */
176	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
177	do { \
178	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
179	HMGetVmxExitName(a_uExitReason))); \
180	} while (0) \
181
182
183	/*********************************************************************************************************************************
184	* Structures and Typedefs *
185	*********************************************************************************************************************************/
186	/**
187	* VMX per-VCPU transient state.
188	*
189	* A state structure for holding miscellaneous information across
190	* VMX non-root operation and restored after the transition.
191	*
192	* Note: The members are ordered and aligned such that the most
193	* frequently used ones (in the guest execution loop) fall within
194	* the first cache line.
195	*/
196	typedef struct VMXTRANSIENT
197	{
198	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
199	uint32_t fVmcsFieldsRead;
200	/** The guest's TPR value used for TPR shadowing. */
201	uint8_t u8GuestTpr;
202	uint8_t abAlignment0[3];
203
204	/** Whether the VM-exit was caused by a page-fault during delivery of an
205	* external interrupt or NMI. */
206	bool fVectoringPF;
207	/** Whether the VM-exit was caused by a page-fault during delivery of a
208	* contributory exception or a page-fault. */
209	bool fVectoringDoublePF;
210	/** Whether the VM-entry failed or not. */
211	bool fVMEntryFailed;
212	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
213	* area after VM-exit. */
214	bool fRemoveTscAuxMsr;
215	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
216	bool fUpdatedTscOffsettingAndPreemptTimer;
217	/** Whether we are currently executing a nested-guest. */
218	bool fIsNestedGuest;
219	/** Whether the guest debug state was active at the time of VM-exit. */
220	bool fWasGuestDebugStateActive;
221	/** Whether the hyper debug state was active at the time of VM-exit. */
222	bool fWasHyperDebugStateActive;
223
224	/** The basic VM-exit reason. */
225	uint32_t uExitReason;
226	/** The VM-exit interruption error code. */
227	uint32_t uExitIntErrorCode;
228
229	/** The host's rflags/eflags. */
230	RTCCUINTREG fEFlags;
231
232	/** The VM-exit exit code qualification. */
233	uint64_t uExitQual;
234
235	/** The VMCS info. object. */
236	PVMXVMCSINFO pVmcsInfo;
237
238	/** The VM-exit interruption-information field. */
239	uint32_t uExitIntInfo;
240	/** The VM-exit instruction-length field. */
241	uint32_t cbExitInstr;
242
243	/** The VM-exit instruction-information field. */
244	VMXEXITINSTRINFO ExitInstrInfo;
245	/** IDT-vectoring information field. */
246	uint32_t uIdtVectoringInfo;
247
248	/** IDT-vectoring error code. */
249	uint32_t uIdtVectoringErrorCode;
250	uint32_t u32Alignment0;
251
252	/** The Guest-linear address. */
253	uint64_t uGuestLinearAddr;
254
255	/** The Guest-physical address. */
256	uint64_t uGuestPhysicalAddr;
257
258	/** The Guest pending-debug exceptions. */
259	uint64_t uGuestPendingDbgXcpts;
260
261	/** The VM-entry interruption-information field. */
262	uint32_t uEntryIntInfo;
263	/** The VM-entry exception error code field. */
264	uint32_t uEntryXcptErrorCode;
265
266	/** The VM-entry instruction length field. */
267	uint32_t cbEntryInstr;
268	} VMXTRANSIENT;
269	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, fVmcsFieldsRead, 8);
271	AssertCompileMemberAlignment(VMXTRANSIENT, fVectoringPF, 8);
272	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, 8);
273	AssertCompileMemberAlignment(VMXTRANSIENT, fEFlags, 8);
274	AssertCompileMemberAlignment(VMXTRANSIENT, uExitQual, 8);
275	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, 8);
276	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, 8);
277	AssertCompileMemberAlignment(VMXTRANSIENT, ExitInstrInfo, 8);
278	AssertCompileMemberAlignment(VMXTRANSIENT, uIdtVectoringErrorCode, 8);
279	AssertCompileMemberAlignment(VMXTRANSIENT, uGuestLinearAddr, 8);
280	AssertCompileMemberAlignment(VMXTRANSIENT, uGuestPhysicalAddr, 8);
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, 8);
282	AssertCompileMemberAlignment(VMXTRANSIENT, cbEntryInstr, 8);
283	/** Pointer to VMX transient state. */
284	typedef VMXTRANSIENT *PVMXTRANSIENT;
285	/** Pointer to a const VMX transient state. */
286	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
287
288	/**
289	* VMX page allocation information.
290	*/
291	typedef struct
292	{
293	uint32_t fValid; /*< Whether to allocate this page (e.g, based on a CPU feature). /
294	uint32_t uPadding0; /*< Padding to ensure array of these structs are aligned to a multiple of 8. /
295	PRTHCPHYS pHCPhys; /*< Where to store the host-physical address of the allocation. /
296	PRTR0PTR ppVirt; /*< Where to store the host-virtual address of the allocation. /
297	} VMXPAGEALLOCINFO;
298	/** Pointer to VMX page-allocation info. */
299	typedef VMXPAGEALLOCINFO *PVMXPAGEALLOCINFO;
300	/** Pointer to a const VMX page-allocation info. */
301	typedef const VMXPAGEALLOCINFO *PCVMXPAGEALLOCINFO;
302	AssertCompileSizeAlignment(VMXPAGEALLOCINFO, 8);
303
304	/**
305	* Memory operand read or write access.
306	*/
307	typedef enum VMXMEMACCESS
308	{
309	VMXMEMACCESS_READ = 0,
310	VMXMEMACCESS_WRITE = 1
311	} VMXMEMACCESS;
312
313	/**
314	* VMX VM-exit handler.
315	*
316	* @returns Strict VBox status code (i.e. informational status codes too).
317	* @param pVCpu The cross context virtual CPU structure.
318	* @param pVmxTransient The VMX-transient structure.
319	*/
320	#ifndef HMVMX_USE_FUNCTION_TABLE
321	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
322	#else
323	typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
324	/** Pointer to VM-exit handler. */
325	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
326	#endif
327
328	/**
329	* VMX VM-exit handler, non-strict status code.
330	*
331	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
332	*
333	* @returns VBox status code, no informational status code returned.
334	* @param pVCpu The cross context virtual CPU structure.
335	* @param pVmxTransient The VMX-transient structure.
336	*
337	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
338	* use of that status code will be replaced with VINF_EM_SOMETHING
339	* later when switching over to IEM.
340	*/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
343	#else
344	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
345	#endif
346
347
348	/*********************************************************************************************************************************
349	* Internal Functions *
350	*********************************************************************************************************************************/
351	#ifndef HMVMX_USE_FUNCTION_TABLE
352	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
353	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
354	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
355	#else
356	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
357	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
358	#endif
359	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
360	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
361	#endif
362
363	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
364
365	/** @name VM-exit handler prototypes.
366	* @{
367	*/
368	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
369	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
370	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
371	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
372	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
373	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
374	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
375	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
376	static FNVMXEXITHANDLER hmR0VmxExitHlt;
377	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
378	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
379	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
380	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
381	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
382	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
383	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
384	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
385	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
386	static FNVMXEXITHANDLER hmR0VmxExitVmread;
387	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
388	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
389	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
390	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
391	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
392	#endif
393	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
394	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
395	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
396	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
397	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
398	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
399	static FNVMXEXITHANDLER hmR0VmxExitMwait;
400	static FNVMXEXITHANDLER hmR0VmxExitMtf;
401	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
402	static FNVMXEXITHANDLER hmR0VmxExitPause;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
404	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
405	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
406	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
407	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
408	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
410	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
411	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
415	/** @} */
416
417	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
418	/** @name Nested-guest VM-exit handler prototypes.
419	* @{
420	*/
421	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
422	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
423	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
424	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
425	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
426	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
427	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
428	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
429	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
430	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
431	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
432	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
433	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
434	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
435	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
436	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
437	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
438	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
439	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
440	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
441	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
442	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
443	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
444	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
445	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
446	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
447	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
448	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
449	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
450	/** @} */
451	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
452
453
454	/*********************************************************************************************************************************
455	* Global Variables *
456	*********************************************************************************************************************************/
457	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
458	/**
459	* Array of all VMCS fields.
460	* Any fields added to the VT-x spec. should be added here.
461	*
462	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
463	* of nested-guests.
464	*/
465	static const uint32_t g_aVmcsFields[] =
466	{
467	/* 16-bit control fields. */
468	VMX_VMCS16_VPID,
469	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
470	VMX_VMCS16_EPTP_INDEX,
471
472	/* 16-bit guest-state fields. */
473	VMX_VMCS16_GUEST_ES_SEL,
474	VMX_VMCS16_GUEST_CS_SEL,
475	VMX_VMCS16_GUEST_SS_SEL,
476	VMX_VMCS16_GUEST_DS_SEL,
477	VMX_VMCS16_GUEST_FS_SEL,
478	VMX_VMCS16_GUEST_GS_SEL,
479	VMX_VMCS16_GUEST_LDTR_SEL,
480	VMX_VMCS16_GUEST_TR_SEL,
481	VMX_VMCS16_GUEST_INTR_STATUS,
482	VMX_VMCS16_GUEST_PML_INDEX,
483
484	/* 16-bits host-state fields. */
485	VMX_VMCS16_HOST_ES_SEL,
486	VMX_VMCS16_HOST_CS_SEL,
487	VMX_VMCS16_HOST_SS_SEL,
488	VMX_VMCS16_HOST_DS_SEL,
489	VMX_VMCS16_HOST_FS_SEL,
490	VMX_VMCS16_HOST_GS_SEL,
491	VMX_VMCS16_HOST_TR_SEL,
492
493	/* 64-bit control fields. */
494	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
495	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
496	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
497	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
498	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
499	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
500	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
501	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
502	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
503	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
504	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
505	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
506	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
507	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
508	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
509	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
510	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
511	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
512	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
513	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
514	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
515	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
516	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
517	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
518	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
519	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
520	VMX_VMCS64_CTRL_EPTP_FULL,
521	VMX_VMCS64_CTRL_EPTP_HIGH,
522	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
523	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
524	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
525	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
526	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
527	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
528	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
529	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
530	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
531	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
532	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
533	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
534	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
535	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
536	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
537	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
538	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
539	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
540	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
541	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
542	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
543	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
544
545	/* 64-bit read-only data fields. */
546	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
547	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
548
549	/* 64-bit guest-state fields. */
550	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
551	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
552	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
553	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
554	VMX_VMCS64_GUEST_PAT_FULL,
555	VMX_VMCS64_GUEST_PAT_HIGH,
556	VMX_VMCS64_GUEST_EFER_FULL,
557	VMX_VMCS64_GUEST_EFER_HIGH,
558	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
559	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
560	VMX_VMCS64_GUEST_PDPTE0_FULL,
561	VMX_VMCS64_GUEST_PDPTE0_HIGH,
562	VMX_VMCS64_GUEST_PDPTE1_FULL,
563	VMX_VMCS64_GUEST_PDPTE1_HIGH,
564	VMX_VMCS64_GUEST_PDPTE2_FULL,
565	VMX_VMCS64_GUEST_PDPTE2_HIGH,
566	VMX_VMCS64_GUEST_PDPTE3_FULL,
567	VMX_VMCS64_GUEST_PDPTE3_HIGH,
568	VMX_VMCS64_GUEST_BNDCFGS_FULL,
569	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
570
571	/* 64-bit host-state fields. */
572	VMX_VMCS64_HOST_PAT_FULL,
573	VMX_VMCS64_HOST_PAT_HIGH,
574	VMX_VMCS64_HOST_EFER_FULL,
575	VMX_VMCS64_HOST_EFER_HIGH,
576	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
577	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
578
579	/* 32-bit control fields. */
580	VMX_VMCS32_CTRL_PIN_EXEC,
581	VMX_VMCS32_CTRL_PROC_EXEC,
582	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
583	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
584	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
585	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
586	VMX_VMCS32_CTRL_EXIT,
587	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
588	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
589	VMX_VMCS32_CTRL_ENTRY,
590	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
591	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
592	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
593	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
594	VMX_VMCS32_CTRL_TPR_THRESHOLD,
595	VMX_VMCS32_CTRL_PROC_EXEC2,
596	VMX_VMCS32_CTRL_PLE_GAP,
597	VMX_VMCS32_CTRL_PLE_WINDOW,
598
599	/* 32-bits read-only fields. */
600	VMX_VMCS32_RO_VM_INSTR_ERROR,
601	VMX_VMCS32_RO_EXIT_REASON,
602	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
603	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
604	VMX_VMCS32_RO_IDT_VECTORING_INFO,
605	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
606	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
607	VMX_VMCS32_RO_EXIT_INSTR_INFO,
608
609	/* 32-bit guest-state fields. */
610	VMX_VMCS32_GUEST_ES_LIMIT,
611	VMX_VMCS32_GUEST_CS_LIMIT,
612	VMX_VMCS32_GUEST_SS_LIMIT,
613	VMX_VMCS32_GUEST_DS_LIMIT,
614	VMX_VMCS32_GUEST_FS_LIMIT,
615	VMX_VMCS32_GUEST_GS_LIMIT,
616	VMX_VMCS32_GUEST_LDTR_LIMIT,
617	VMX_VMCS32_GUEST_TR_LIMIT,
618	VMX_VMCS32_GUEST_GDTR_LIMIT,
619	VMX_VMCS32_GUEST_IDTR_LIMIT,
620	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
621	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
622	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
623	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
624	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
625	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
626	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
627	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
628	VMX_VMCS32_GUEST_INT_STATE,
629	VMX_VMCS32_GUEST_ACTIVITY_STATE,
630	VMX_VMCS32_GUEST_SMBASE,
631	VMX_VMCS32_GUEST_SYSENTER_CS,
632	VMX_VMCS32_PREEMPT_TIMER_VALUE,
633
634	/* 32-bit host-state fields. */
635	VMX_VMCS32_HOST_SYSENTER_CS,
636
637	/* Natural-width control fields. */
638	VMX_VMCS_CTRL_CR0_MASK,
639	VMX_VMCS_CTRL_CR4_MASK,
640	VMX_VMCS_CTRL_CR0_READ_SHADOW,
641	VMX_VMCS_CTRL_CR4_READ_SHADOW,
642	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
643	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
644	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
645	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
646
647	/* Natural-width read-only data fields. */
648	VMX_VMCS_RO_EXIT_QUALIFICATION,
649	VMX_VMCS_RO_IO_RCX,
650	VMX_VMCS_RO_IO_RSI,
651	VMX_VMCS_RO_IO_RDI,
652	VMX_VMCS_RO_IO_RIP,
653	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
654
655	/* Natural-width guest-state field */
656	VMX_VMCS_GUEST_CR0,
657	VMX_VMCS_GUEST_CR3,
658	VMX_VMCS_GUEST_CR4,
659	VMX_VMCS_GUEST_ES_BASE,
660	VMX_VMCS_GUEST_CS_BASE,
661	VMX_VMCS_GUEST_SS_BASE,
662	VMX_VMCS_GUEST_DS_BASE,
663	VMX_VMCS_GUEST_FS_BASE,
664	VMX_VMCS_GUEST_GS_BASE,
665	VMX_VMCS_GUEST_LDTR_BASE,
666	VMX_VMCS_GUEST_TR_BASE,
667	VMX_VMCS_GUEST_GDTR_BASE,
668	VMX_VMCS_GUEST_IDTR_BASE,
669	VMX_VMCS_GUEST_DR7,
670	VMX_VMCS_GUEST_RSP,
671	VMX_VMCS_GUEST_RIP,
672	VMX_VMCS_GUEST_RFLAGS,
673	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
674	VMX_VMCS_GUEST_SYSENTER_ESP,
675	VMX_VMCS_GUEST_SYSENTER_EIP,
676
677	/* Natural-width host-state fields */
678	VMX_VMCS_HOST_CR0,
679	VMX_VMCS_HOST_CR3,
680	VMX_VMCS_HOST_CR4,
681	VMX_VMCS_HOST_FS_BASE,
682	VMX_VMCS_HOST_GS_BASE,
683	VMX_VMCS_HOST_TR_BASE,
684	VMX_VMCS_HOST_GDTR_BASE,
685	VMX_VMCS_HOST_IDTR_BASE,
686	VMX_VMCS_HOST_SYSENTER_ESP,
687	VMX_VMCS_HOST_SYSENTER_EIP,
688	VMX_VMCS_HOST_RSP,
689	VMX_VMCS_HOST_RIP
690	};
691	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
692
693	#ifdef VBOX_STRICT
694	static const uint32_t g_aVmcsSegBase[] =
695	{
696	VMX_VMCS_GUEST_ES_BASE,
697	VMX_VMCS_GUEST_CS_BASE,
698	VMX_VMCS_GUEST_SS_BASE,
699	VMX_VMCS_GUEST_DS_BASE,
700	VMX_VMCS_GUEST_FS_BASE,
701	VMX_VMCS_GUEST_GS_BASE
702	};
703	static const uint32_t g_aVmcsSegSel[] =
704	{
705	VMX_VMCS16_GUEST_ES_SEL,
706	VMX_VMCS16_GUEST_CS_SEL,
707	VMX_VMCS16_GUEST_SS_SEL,
708	VMX_VMCS16_GUEST_DS_SEL,
709	VMX_VMCS16_GUEST_FS_SEL,
710	VMX_VMCS16_GUEST_GS_SEL
711	};
712	static const uint32_t g_aVmcsSegLimit[] =
713	{
714	VMX_VMCS32_GUEST_ES_LIMIT,
715	VMX_VMCS32_GUEST_CS_LIMIT,
716	VMX_VMCS32_GUEST_SS_LIMIT,
717	VMX_VMCS32_GUEST_DS_LIMIT,
718	VMX_VMCS32_GUEST_FS_LIMIT,
719	VMX_VMCS32_GUEST_GS_LIMIT
720	};
721	static const uint32_t g_aVmcsSegAttr[] =
722	{
723	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
724	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
725	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
726	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
727	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
728	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
729	};
730	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
731	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
732	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
733	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
734	#endif /* VBOX_STRICT */
735
736	#ifdef HMVMX_USE_FUNCTION_TABLE
737	/**
738	* VMX_EXIT dispatch table.
739	*/
740	static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
741	{
742	/* 0 VMX_EXIT_XCPT_OR_NMI */ { hmR0VmxExitXcptOrNmi },
743	/* 1 VMX_EXIT_EXT_INT */ { hmR0VmxExitExtInt },
744	/* 2 VMX_EXIT_TRIPLE_FAULT */ { hmR0VmxExitTripleFault },
745	/* 3 VMX_EXIT_INIT_SIGNAL */ { hmR0VmxExitErrUnexpected },
746	/* 4 VMX_EXIT_SIPI */ { hmR0VmxExitErrUnexpected },
747	/* 5 VMX_EXIT_IO_SMI */ { hmR0VmxExitErrUnexpected },
748	/* 6 VMX_EXIT_SMI */ { hmR0VmxExitErrUnexpected },
749	/* 7 VMX_EXIT_INT_WINDOW */ { hmR0VmxExitIntWindow },
750	/* 8 VMX_EXIT_NMI_WINDOW */ { hmR0VmxExitNmiWindow },
751	/* 9 VMX_EXIT_TASK_SWITCH */ { hmR0VmxExitTaskSwitch },
752	/* 10 VMX_EXIT_CPUID */ { hmR0VmxExitCpuid },
753	/* 11 VMX_EXIT_GETSEC */ { hmR0VmxExitGetsec },
754	/* 12 VMX_EXIT_HLT */ { hmR0VmxExitHlt },
755	/* 13 VMX_EXIT_INVD */ { hmR0VmxExitInvd },
756	/* 14 VMX_EXIT_INVLPG */ { hmR0VmxExitInvlpg },
757	/* 15 VMX_EXIT_RDPMC */ { hmR0VmxExitRdpmc },
758	/* 16 VMX_EXIT_RDTSC */ { hmR0VmxExitRdtsc },
759	/* 17 VMX_EXIT_RSM */ { hmR0VmxExitErrUnexpected },
760	/* 18 VMX_EXIT_VMCALL */ { hmR0VmxExitVmcall },
761	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
762	/* 19 VMX_EXIT_VMCLEAR */ { hmR0VmxExitVmclear },
763	/* 20 VMX_EXIT_VMLAUNCH */ { hmR0VmxExitVmlaunch },
764	/* 21 VMX_EXIT_VMPTRLD */ { hmR0VmxExitVmptrld },
765	/* 22 VMX_EXIT_VMPTRST */ { hmR0VmxExitVmptrst },
766	/* 23 VMX_EXIT_VMREAD */ { hmR0VmxExitVmread },
767	/* 24 VMX_EXIT_VMRESUME */ { hmR0VmxExitVmresume },
768	/* 25 VMX_EXIT_VMWRITE */ { hmR0VmxExitVmwrite },
769	/* 26 VMX_EXIT_VMXOFF */ { hmR0VmxExitVmxoff },
770	/* 27 VMX_EXIT_VMXON */ { hmR0VmxExitVmxon },
771	#else
772	/* 19 VMX_EXIT_VMCLEAR */ { hmR0VmxExitSetPendingXcptUD },
773	/* 20 VMX_EXIT_VMLAUNCH */ { hmR0VmxExitSetPendingXcptUD },
774	/* 21 VMX_EXIT_VMPTRLD */ { hmR0VmxExitSetPendingXcptUD },
775	/* 22 VMX_EXIT_VMPTRST */ { hmR0VmxExitSetPendingXcptUD },
776	/* 23 VMX_EXIT_VMREAD */ { hmR0VmxExitSetPendingXcptUD },
777	/* 24 VMX_EXIT_VMRESUME */ { hmR0VmxExitSetPendingXcptUD },
778	/* 25 VMX_EXIT_VMWRITE */ { hmR0VmxExitSetPendingXcptUD },
779	/* 26 VMX_EXIT_VMXOFF */ { hmR0VmxExitSetPendingXcptUD },
780	/* 27 VMX_EXIT_VMXON */ { hmR0VmxExitSetPendingXcptUD },
781	#endif
782	/* 28 VMX_EXIT_MOV_CRX */ { hmR0VmxExitMovCRx },
783	/* 29 VMX_EXIT_MOV_DRX */ { hmR0VmxExitMovDRx },
784	/* 30 VMX_EXIT_IO_INSTR */ { hmR0VmxExitIoInstr },
785	/* 31 VMX_EXIT_RDMSR */ { hmR0VmxExitRdmsr },
786	/* 32 VMX_EXIT_WRMSR */ { hmR0VmxExitWrmsr },
787	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ { hmR0VmxExitErrInvalidGuestState },
788	/* 34 VMX_EXIT_ERR_MSR_LOAD */ { hmR0VmxExitErrUnexpected },
789	/* 35 UNDEFINED */ { hmR0VmxExitErrUnexpected },
790	/* 36 VMX_EXIT_MWAIT */ { hmR0VmxExitMwait },
791	/* 37 VMX_EXIT_MTF */ { hmR0VmxExitMtf },
792	/* 38 UNDEFINED */ { hmR0VmxExitErrUnexpected },
793	/* 39 VMX_EXIT_MONITOR */ { hmR0VmxExitMonitor },
794	/* 40 VMX_EXIT_PAUSE */ { hmR0VmxExitPause },
795	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ { hmR0VmxExitErrUnexpected },
796	/* 42 UNDEFINED */ { hmR0VmxExitErrUnexpected },
797	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ { hmR0VmxExitTprBelowThreshold },
798	/* 44 VMX_EXIT_APIC_ACCESS */ { hmR0VmxExitApicAccess },
799	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ { hmR0VmxExitErrUnexpected },
800	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ { hmR0VmxExitErrUnexpected },
801	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ { hmR0VmxExitErrUnexpected },
802	/* 48 VMX_EXIT_EPT_VIOLATION */ { hmR0VmxExitEptViolation },
803	/* 49 VMX_EXIT_EPT_MISCONFIG */ { hmR0VmxExitEptMisconfig },
804	/* 50 VMX_EXIT_INVEPT */ { hmR0VmxExitSetPendingXcptUD },
805	/* 51 VMX_EXIT_RDTSCP */ { hmR0VmxExitRdtscp },
806	/* 52 VMX_EXIT_PREEMPT_TIMER */ { hmR0VmxExitPreemptTimer },
807	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
808	/* 53 VMX_EXIT_INVVPID */ { hmR0VmxExitInvvpid },
809	#else
810	/* 53 VMX_EXIT_INVVPID */ { hmR0VmxExitSetPendingXcptUD },
811	#endif
812	/* 54 VMX_EXIT_WBINVD */ { hmR0VmxExitWbinvd },
813	/* 55 VMX_EXIT_XSETBV */ { hmR0VmxExitXsetbv },
814	/* 56 VMX_EXIT_APIC_WRITE */ { hmR0VmxExitErrUnexpected },
815	/* 57 VMX_EXIT_RDRAND */ { hmR0VmxExitErrUnexpected },
816	/* 58 VMX_EXIT_INVPCID */ { hmR0VmxExitInvpcid },
817	/* 59 VMX_EXIT_VMFUNC */ { hmR0VmxExitErrUnexpected },
818	/* 60 VMX_EXIT_ENCLS */ { hmR0VmxExitErrUnexpected },
819	/* 61 VMX_EXIT_RDSEED */ { hmR0VmxExitErrUnexpected },
820	/* 62 VMX_EXIT_PML_FULL */ { hmR0VmxExitErrUnexpected },
821	/* 63 VMX_EXIT_XSAVES */ { hmR0VmxExitErrUnexpected },
822	/* 64 VMX_EXIT_XRSTORS */ { hmR0VmxExitErrUnexpected },
823	/* 65 UNDEFINED */ { hmR0VmxExitErrUnexpected },
824	/* 66 VMX_EXIT_SPP_EVENT */ { hmR0VmxExitErrUnexpected },
825	/* 67 VMX_EXIT_UMWAIT */ { hmR0VmxExitErrUnexpected },
826	/* 68 VMX_EXIT_TPAUSE */ { hmR0VmxExitErrUnexpected },
827	};
828	#endif /* HMVMX_USE_FUNCTION_TABLE */
829
830	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
831	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
832	{
833	/* 0 */ "(Not Used)",
834	/* 1 */ "VMCALL executed in VMX root operation.",
835	/* 2 */ "VMCLEAR with invalid physical address.",
836	/* 3 */ "VMCLEAR with VMXON pointer.",
837	/* 4 */ "VMLAUNCH with non-clear VMCS.",
838	/* 5 */ "VMRESUME with non-launched VMCS.",
839	/* 6 */ "VMRESUME after VMXOFF",
840	/* 7 */ "VM-entry with invalid control fields.",
841	/* 8 */ "VM-entry with invalid host state fields.",
842	/* 9 */ "VMPTRLD with invalid physical address.",
843	/* 10 */ "VMPTRLD with VMXON pointer.",
844	/* 11 */ "VMPTRLD with incorrect revision identifier.",
845	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
846	/* 13 */ "VMWRITE to read-only VMCS component.",
847	/* 14 */ "(Not Used)",
848	/* 15 */ "VMXON executed in VMX root operation.",
849	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
850	/* 17 */ "VM-entry with non-launched executing VMCS.",
851	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
852	/* 19 */ "VMCALL with non-clear VMCS.",
853	/* 20 */ "VMCALL with invalid VM-exit control fields.",
854	/* 21 */ "(Not Used)",
855	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
856	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
857	/* 24 */ "VMCALL with invalid SMM-monitor features.",
858	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
859	/* 26 */ "VM-entry with events blocked by MOV SS.",
860	/* 27 */ "(Not Used)",
861	/* 28 */ "Invalid operand to INVEPT/INVVPID."
862	};
863	#endif /* VBOX_STRICT && LOG_ENABLED */
864
865
866	/**
867	* Checks if the given MSR is part of the lastbranch-from-IP MSR stack.
868	* @returns @c true if it's part of LBR stack, @c false otherwise.
869	*
870	* @param pVM The cross context VM structure.
871	* @param idMsr The MSR.
872	* @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
873	* Optional, can be NULL.
874	*
875	* @remarks Must only be called when LBR is enabled.
876	*/
877	DECL_FORCE_INLINE(bool) hmR0VmxIsLbrBranchFromMsr(PCVM pVM, uint32_t idMsr, uint32_t *pidxMsr)
878	{
879	Assert(pVM->hm.s.vmx.fLbr);
880	Assert(pVM->hm.s.vmx.idLbrFromIpMsrFirst);
881	uint32_t const cLbrStack = pVM->hm.s.vmx.idLbrFromIpMsrLast - pVM->hm.s.vmx.idLbrFromIpMsrFirst + 1;
882	uint32_t const idxMsr = idMsr - pVM->hm.s.vmx.idLbrFromIpMsrFirst;
883	if (idxMsr < cLbrStack)
884	{
885	if (pidxMsr)
886	*pidxMsr = idxMsr;
887	return true;
888	}
889	return false;
890	}
891
892
893	/**
894	* Checks if the given MSR is part of the lastbranch-to-IP MSR stack.
895	* @returns @c true if it's part of LBR stack, @c false otherwise.
896	*
897	* @param pVM The cross context VM structure.
898	* @param idMsr The MSR.
899	* @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
900	* Optional, can be NULL.
901	*
902	* @remarks Must only be called when LBR is enabled and when lastbranch-to-IP MSRs
903	* are supported by the CPU (see hmR0VmxSetupLbrMsrRange).
904	*/
905	DECL_FORCE_INLINE(bool) hmR0VmxIsLbrBranchToMsr(PCVM pVM, uint32_t idMsr, uint32_t *pidxMsr)
906	{
907	Assert(pVM->hm.s.vmx.fLbr);
908	if (pVM->hm.s.vmx.idLbrToIpMsrFirst)
909	{
910	uint32_t const cLbrStack = pVM->hm.s.vmx.idLbrToIpMsrLast - pVM->hm.s.vmx.idLbrToIpMsrFirst + 1;
911	uint32_t const idxMsr = idMsr - pVM->hm.s.vmx.idLbrToIpMsrFirst;
912	if (idxMsr < cLbrStack)
913	{
914	if (pidxMsr)
915	*pidxMsr = idxMsr;
916	return true;
917	}
918	}
919	return false;
920	}
921
922
923	/**
924	* Gets the CR0 guest/host mask.
925	*
926	* These bits typically does not change through the lifetime of a VM. Any bit set in
927	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
928	* by the guest.
929	*
930	* @returns The CR0 guest/host mask.
931	* @param pVCpu The cross context virtual CPU structure.
932	*/
933	static uint64_t hmR0VmxGetFixedCr0Mask(PCVMCPUCC pVCpu)
934	{
935	/*
936	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
937	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
938	*
939	* Furthermore, modifications to any bits that are reserved/unspecified currently
940	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
941	* when future CPUs specify and use currently reserved/unspecified bits.
942	*/
943	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
944	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
945	* and @bugref{6944}. */
946	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
947	return ( X86_CR0_PE
948	\| X86_CR0_NE
949	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
950	\| X86_CR0_PG
951	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
952	}
953
954
955	/**
956	* Gets the CR4 guest/host mask.
957	*
958	* These bits typically does not change through the lifetime of a VM. Any bit set in
959	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
960	* by the guest.
961	*
962	* @returns The CR4 guest/host mask.
963	* @param pVCpu The cross context virtual CPU structure.
964	*/
965	static uint64_t hmR0VmxGetFixedCr4Mask(PCVMCPUCC pVCpu)
966	{
967	/*
968	* We construct a mask of all CR4 bits that the guest can modify without causing
969	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
970	* a VM-exit when the guest attempts to modify them when executing using
971	* hardware-assisted VMX.
972	*
973	* When a feature is not exposed to the guest (and may be present on the host),
974	* we want to intercept guest modifications to the bit so we can emulate proper
975	* behavior (e.g., #GP).
976	*
977	* Furthermore, only modifications to those bits that don't require immediate
978	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
979	* depends on CR3 which might not always be the guest value while executing
980	* using hardware-assisted VMX.
981	*/
982	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
983	bool const fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
984	bool const fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
985	bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
986
987	/*
988	* Paranoia.
989	* Ensure features exposed to the guest are present on the host.
990	*/
991	Assert(!fFsGsBase \|\| pVM->cpum.ro.HostFeatures.fFsGsBase);
992	Assert(!fXSaveRstor \|\| pVM->cpum.ro.HostFeatures.fXSaveRstor);
993	Assert(!fFxSaveRstor \|\| pVM->cpum.ro.HostFeatures.fFxSaveRstor);
994
995	uint64_t const fGstMask = ( X86_CR4_PVI
996	\| X86_CR4_TSD
997	\| X86_CR4_DE
998	\| X86_CR4_MCE
999	\| X86_CR4_PCE
1000	\| X86_CR4_OSXMMEEXCPT
1001	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
1002	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
1003	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0));
1004	return ~fGstMask;
1005	}
1006
1007
1008	/**
1009	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
1010	* area.
1011	*
1012	* @returns @c true if it's different, @c false otherwise.
1013	* @param pVmcsInfo The VMCS info. object.
1014	*/
1015	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
1016	{
1017	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
1018	&& pVmcsInfo->pvGuestMsrStore);
1019	}
1020
1021
1022	/**
1023	* Sets the given Processor-based VM-execution controls.
1024	*
1025	* @param pVmxTransient The VMX-transient structure.
1026	* @param uProcCtls The Processor-based VM-execution controls to set.
1027	*/
1028	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
1029	{
1030	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1031	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
1032	{
1033	pVmcsInfo->u32ProcCtls \|= uProcCtls;
1034	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
1035	AssertRC(rc);
1036	}
1037	}
1038
1039
1040	/**
1041	* Removes the given Processor-based VM-execution controls.
1042	*
1043	* @param pVCpu The cross context virtual CPU structure.
1044	* @param pVmxTransient The VMX-transient structure.
1045	* @param uProcCtls The Processor-based VM-execution controls to remove.
1046	*
1047	* @remarks When executing a nested-guest, this will not remove any of the specified
1048	* controls if the nested hypervisor has set any one of them.
1049	*/
1050	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
1051	{
1052	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1053	if (pVmcsInfo->u32ProcCtls & uProcCtls)
1054	{
1055	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1056	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
1057	? true
1058	: !CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uProcCtls);
1059	#else
1060	NOREF(pVCpu);
1061	bool const fRemoveCtls = true;
1062	#endif
1063	if (fRemoveCtls)
1064	{
1065	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
1066	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
1067	AssertRC(rc);
1068	}
1069	}
1070	}
1071
1072
1073	/**
1074	* Sets the TSC offset for the current VMCS.
1075	*
1076	* @param uTscOffset The TSC offset to set.
1077	* @param pVmcsInfo The VMCS info. object.
1078	*/
1079	static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
1080	{
1081	if (pVmcsInfo->u64TscOffset != uTscOffset)
1082	{
1083	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
1084	AssertRC(rc);
1085	pVmcsInfo->u64TscOffset = uTscOffset;
1086	}
1087	}
1088
1089
1090	/**
1091	* Adds one or more exceptions to the exception bitmap and commits it to the current
1092	* VMCS.
1093	*
1094	* @param pVmxTransient The VMX-transient structure.
1095	* @param uXcptMask The exception(s) to add.
1096	*/
1097	static void hmR0VmxAddXcptInterceptMask(PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1098	{
1099	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1100	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1101	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1102	{
1103	uXcptBitmap \|= uXcptMask;
1104	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1105	AssertRC(rc);
1106	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1107	}
1108	}
1109
1110
1111	/**
1112	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1113	*
1114	* @param pVmxTransient The VMX-transient structure.
1115	* @param uXcpt The exception to add.
1116	*/
1117	static void hmR0VmxAddXcptIntercept(PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1118	{
1119	Assert(uXcpt <= X86_XCPT_LAST);
1120	hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1121	}
1122
1123
1124	/**
1125	* Remove one or more exceptions from the exception bitmap and commits it to the
1126	* current VMCS.
1127	*
1128	* This takes care of not removing the exception intercept if a nested-guest
1129	* requires the exception to be intercepted.
1130	*
1131	* @returns VBox status code.
1132	* @param pVCpu The cross context virtual CPU structure.
1133	* @param pVmxTransient The VMX-transient structure.
1134	* @param uXcptMask The exception(s) to remove.
1135	*/
1136	static int hmR0VmxRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1137	{
1138	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1139	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1140	if (u32XcptBitmap & uXcptMask)
1141	{
1142	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1143	if (!pVmxTransient->fIsNestedGuest)
1144	{ /* likely */ }
1145	else
1146	{
1147	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1148	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1149	}
1150	#endif
1151	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1152	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1153	\| RT_BIT(X86_XCPT_DE)
1154	\| RT_BIT(X86_XCPT_NM)
1155	\| RT_BIT(X86_XCPT_TS)
1156	\| RT_BIT(X86_XCPT_UD)
1157	\| RT_BIT(X86_XCPT_NP)
1158	\| RT_BIT(X86_XCPT_SS)
1159	\| RT_BIT(X86_XCPT_GP)
1160	\| RT_BIT(X86_XCPT_PF)
1161	\| RT_BIT(X86_XCPT_MF));
1162	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1163	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1164	#endif
1165	if (uXcptMask)
1166	{
1167	/* Validate we are not removing any essential exception intercepts. */
1168	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1169	NOREF(pVCpu);
1170	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1171	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1172
1173	/* Remove it from the exception bitmap. */
1174	u32XcptBitmap &= ~uXcptMask;
1175
1176	/* Commit and update the cache if necessary. */
1177	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1178	{
1179	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1180	AssertRC(rc);
1181	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1182	}
1183	}
1184	}
1185	return VINF_SUCCESS;
1186	}
1187
1188
1189	/**
1190	* Remove an exceptions from the exception bitmap and commits it to the current
1191	* VMCS.
1192	*
1193	* @returns VBox status code.
1194	* @param pVCpu The cross context virtual CPU structure.
1195	* @param pVmxTransient The VMX-transient structure.
1196	* @param uXcpt The exception to remove.
1197	*/
1198	static int hmR0VmxRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1199	{
1200	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1201	}
1202
1203
1204	/**
1205	* Loads the VMCS specified by the VMCS info. object.
1206	*
1207	* @returns VBox status code.
1208	* @param pVmcsInfo The VMCS info. object.
1209	*
1210	* @remarks Can be called with interrupts disabled.
1211	*/
1212	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1213	{
1214	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1215	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1216
1217	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1218	if (RT_SUCCESS(rc))
1219	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1220	return rc;
1221	}
1222
1223
1224	/**
1225	* Clears the VMCS specified by the VMCS info. object.
1226	*
1227	* @returns VBox status code.
1228	* @param pVmcsInfo The VMCS info. object.
1229	*
1230	* @remarks Can be called with interrupts disabled.
1231	*/
1232	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1233	{
1234	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1235	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1236
1237	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1238	if (RT_SUCCESS(rc))
1239	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1240	return rc;
1241	}
1242
1243
1244	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1245	/**
1246	* Loads the shadow VMCS specified by the VMCS info. object.
1247	*
1248	* @returns VBox status code.
1249	* @param pVmcsInfo The VMCS info. object.
1250	*
1251	* @remarks Can be called with interrupts disabled.
1252	*/
1253	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1254	{
1255	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1256	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1257
1258	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1259	if (RT_SUCCESS(rc))
1260	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1261	return rc;
1262	}
1263
1264
1265	/**
1266	* Clears the shadow VMCS specified by the VMCS info. object.
1267	*
1268	* @returns VBox status code.
1269	* @param pVmcsInfo The VMCS info. object.
1270	*
1271	* @remarks Can be called with interrupts disabled.
1272	*/
1273	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1274	{
1275	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1276	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1277
1278	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1279	if (RT_SUCCESS(rc))
1280	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1281	return rc;
1282	}
1283
1284
1285	/**
1286	* Switches from and to the specified VMCSes.
1287	*
1288	* @returns VBox status code.
1289	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1290	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1291	*
1292	* @remarks Called with interrupts disabled.
1293	*/
1294	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1295	{
1296	/*
1297	* Clear the VMCS we are switching out if it has not already been cleared.
1298	* This will sync any CPU internal data back to the VMCS.
1299	*/
1300	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1301	{
1302	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1303	if (RT_SUCCESS(rc))
1304	{
1305	/*
1306	* The shadow VMCS, if any, would not be active at this point since we
1307	* would have cleared it while importing the virtual hardware-virtualization
1308	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1309	* clear the shadow VMCS here, just assert for safety.
1310	*/
1311	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1312	}
1313	else
1314	return rc;
1315	}
1316
1317	/*
1318	* Clear the VMCS we are switching to if it has not already been cleared.
1319	* This will initialize the VMCS launch state to "clear" required for loading it.
1320	*
1321	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1322	*/
1323	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1324	{
1325	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1326	if (RT_SUCCESS(rc))
1327	{ /* likely */ }
1328	else
1329	return rc;
1330	}
1331
1332	/*
1333	* Finally, load the VMCS we are switching to.
1334	*/
1335	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1336	}
1337
1338
1339	/**
1340	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1341	* caller.
1342	*
1343	* @returns VBox status code.
1344	* @param pVCpu The cross context virtual CPU structure.
1345	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1346	* true) or guest VMCS (pass false).
1347	*/
1348	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1349	{
1350	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1351	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1352
1353	PVMXVMCSINFO pVmcsInfoFrom;
1354	PVMXVMCSINFO pVmcsInfoTo;
1355	if (fSwitchToNstGstVmcs)
1356	{
1357	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1358	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1359	}
1360	else
1361	{
1362	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1363	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1364	}
1365
1366	/*
1367	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1368	* preemption hook code path acquires the current VMCS.
1369	*/
1370	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1371
1372	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1373	if (RT_SUCCESS(rc))
1374	{
1375	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1376
1377	/*
1378	* If we are switching to a VMCS that was executed on a different host CPU or was
1379	* never executed before, flag that we need to export the host state before executing
1380	* guest/nested-guest code using hardware-assisted VMX.
1381	*
1382	* This could probably be done in a preemptible context since the preemption hook
1383	* will flag the necessary change in host context. However, since preemption is
1384	* already disabled and to avoid making assumptions about host specific code in
1385	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1386	* disabled.
1387	*/
1388	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1389	{ /* likely */ }
1390	else
1391	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1392
1393	ASMSetFlags(fEFlags);
1394
1395	/*
1396	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1397	* flag that we need to update the host MSR values there. Even if we decide in the
1398	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1399	* if its content differs, we would have to update the host MSRs anyway.
1400	*/
1401	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1402	}
1403	else
1404	ASMSetFlags(fEFlags);
1405	return rc;
1406	}
1407	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1408
1409
1410	/**
1411	* Updates the VM's last error record.
1412	*
1413	* If there was a VMX instruction error, reads the error data from the VMCS and
1414	* updates VCPU's last error record as well.
1415	*
1416	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1417	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1418	* VERR_VMX_INVALID_VMCS_FIELD.
1419	* @param rc The error code.
1420	*/
1421	static void hmR0VmxUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
1422	{
1423	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1424	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1425	{
1426	AssertPtrReturnVoid(pVCpu);
1427	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1428	}
1429	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1430	}
1431
1432
1433	#ifdef VBOX_STRICT
1434	/**
1435	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1436	* transient structure.
1437	*
1438	* @param pVmxTransient The VMX-transient structure.
1439	*/
1440	DECLINLINE(void) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1441	{
1442	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1443	AssertRC(rc);
1444	}
1445
1446
1447	/**
1448	* Reads the VM-entry exception error code field from the VMCS into
1449	* the VMX transient structure.
1450	*
1451	* @param pVmxTransient The VMX-transient structure.
1452	*/
1453	DECLINLINE(void) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1454	{
1455	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1456	AssertRC(rc);
1457	}
1458
1459
1460	/**
1461	* Reads the VM-entry exception error code field from the VMCS into
1462	* the VMX transient structure.
1463	*
1464	* @param pVmxTransient The VMX-transient structure.
1465	*/
1466	DECLINLINE(void) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1467	{
1468	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1469	AssertRC(rc);
1470	}
1471	#endif /* VBOX_STRICT */
1472
1473
1474	/**
1475	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1476	* transient structure.
1477	*
1478	* @param pVmxTransient The VMX-transient structure.
1479	*/
1480	DECLINLINE(void) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1481	{
1482	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1483	{
1484	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1485	AssertRC(rc);
1486	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1487	}
1488	}
1489
1490
1491	/**
1492	* Reads the VM-exit interruption error code from the VMCS into the VMX
1493	* transient structure.
1494	*
1495	* @param pVmxTransient The VMX-transient structure.
1496	*/
1497	DECLINLINE(void) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1498	{
1499	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1500	{
1501	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1502	AssertRC(rc);
1503	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1504	}
1505	}
1506
1507
1508	/**
1509	* Reads the VM-exit instruction length field from the VMCS into the VMX
1510	* transient structure.
1511	*
1512	* @param pVmxTransient The VMX-transient structure.
1513	*/
1514	DECLINLINE(void) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1515	{
1516	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1517	{
1518	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1519	AssertRC(rc);
1520	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1521	}
1522	}
1523
1524
1525	/**
1526	* Reads the VM-exit instruction-information field from the VMCS into
1527	* the VMX transient structure.
1528	*
1529	* @param pVmxTransient The VMX-transient structure.
1530	*/
1531	DECLINLINE(void) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1532	{
1533	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1534	{
1535	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1536	AssertRC(rc);
1537	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1538	}
1539	}
1540
1541
1542	/**
1543	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1544	*
1545	* @param pVmxTransient The VMX-transient structure.
1546	*/
1547	DECLINLINE(void) hmR0VmxReadExitQualVmcs(PVMXTRANSIENT pVmxTransient)
1548	{
1549	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1550	{
1551	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1552	AssertRC(rc);
1553	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1554	}
1555	}
1556
1557
1558	/**
1559	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1560	*
1561	* @param pVmxTransient The VMX-transient structure.
1562	*/
1563	DECLINLINE(void) hmR0VmxReadGuestLinearAddrVmcs(PVMXTRANSIENT pVmxTransient)
1564	{
1565	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1566	{
1567	int rc = VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1568	AssertRC(rc);
1569	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1570	}
1571	}
1572
1573
1574	/**
1575	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1576	*
1577	* @param pVmxTransient The VMX-transient structure.
1578	*/
1579	DECLINLINE(void) hmR0VmxReadGuestPhysicalAddrVmcs(PVMXTRANSIENT pVmxTransient)
1580	{
1581	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1582	{
1583	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1584	AssertRC(rc);
1585	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1586	}
1587	}
1588
1589	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1590	/**
1591	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1592	* structure.
1593	*
1594	* @param pVmxTransient The VMX-transient structure.
1595	*/
1596	DECLINLINE(void) hmR0VmxReadGuestPendingDbgXctps(PVMXTRANSIENT pVmxTransient)
1597	{
1598	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1599	{
1600	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1601	AssertRC(rc);
1602	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1603	}
1604	}
1605	#endif
1606
1607	/**
1608	* Reads the IDT-vectoring information field from the VMCS into the VMX
1609	* transient structure.
1610	*
1611	* @param pVmxTransient The VMX-transient structure.
1612	*
1613	* @remarks No-long-jump zone!!!
1614	*/
1615	DECLINLINE(void) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1616	{
1617	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1618	{
1619	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1620	AssertRC(rc);
1621	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1622	}
1623	}
1624
1625
1626	/**
1627	* Reads the IDT-vectoring error code from the VMCS into the VMX
1628	* transient structure.
1629	*
1630	* @param pVmxTransient The VMX-transient structure.
1631	*/
1632	DECLINLINE(void) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1633	{
1634	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1635	{
1636	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1637	AssertRC(rc);
1638	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1639	}
1640	}
1641
1642	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1643	/**
1644	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1645	*
1646	* @param pVmxTransient The VMX-transient structure.
1647	*/
1648	static void hmR0VmxReadAllRoFieldsVmcs(PVMXTRANSIENT pVmxTransient)
1649	{
1650	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1651	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1652	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1653	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1654	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1655	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1656	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1657	rc \|= VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1658	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1659	AssertRC(rc);
1660	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1661	\| HMVMX_READ_EXIT_INSTR_LEN
1662	\| HMVMX_READ_EXIT_INSTR_INFO
1663	\| HMVMX_READ_IDT_VECTORING_INFO
1664	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1665	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1666	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1667	\| HMVMX_READ_GUEST_LINEAR_ADDR
1668	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1669	}
1670	#endif
1671
1672	/**
1673	* Enters VMX root mode operation on the current CPU.
1674	*
1675	* @returns VBox status code.
1676	* @param pHostCpu The HM physical-CPU structure.
1677	* @param pVM The cross context VM structure. Can be
1678	* NULL, after a resume.
1679	* @param HCPhysCpuPage Physical address of the VMXON region.
1680	* @param pvCpuPage Pointer to the VMXON region.
1681	*/
1682	static int hmR0VmxEnterRootMode(PHMPHYSCPU pHostCpu, PVMCC pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1683	{
1684	Assert(pHostCpu);
1685	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1686	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1687	Assert(pvCpuPage);
1688	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690	if (pVM)
1691	{
1692	/* Write the VMCS revision identifier to the VMXON region. */
1693	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1694	}
1695
1696	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1697	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1698
1699	/* Enable the VMX bit in CR4 if necessary. */
1700	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1701
1702	/* Record whether VMXE was already prior to us enabling it above. */
1703	pHostCpu->fVmxeAlreadyEnabled = RT_BOOL(uOldCr4 & X86_CR4_VMXE);
1704
1705	/* Enter VMX root mode. */
1706	int rc = VMXEnable(HCPhysCpuPage);
1707	if (RT_FAILURE(rc))
1708	{
1709	/* Restore CR4.VMXE if it was not set prior to our attempt to set it above. */
1710	if (!pHostCpu->fVmxeAlreadyEnabled)
1711	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1712
1713	if (pVM)
1714	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1715	}
1716
1717	/* Restore interrupts. */
1718	ASMSetFlags(fEFlags);
1719	return rc;
1720	}
1721
1722
1723	/**
1724	* Exits VMX root mode operation on the current CPU.
1725	*
1726	* @returns VBox status code.
1727	* @param pHostCpu The HM physical-CPU structure.
1728	*/
1729	static int hmR0VmxLeaveRootMode(PHMPHYSCPU pHostCpu)
1730	{
1731	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1732
1733	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1734	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1735
1736	/* If we're for some reason not in VMX root mode, then don't leave it. */
1737	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1738
1739	int rc;
1740	if (uHostCr4 & X86_CR4_VMXE)
1741	{
1742	/* Exit VMX root mode and clear the VMX bit in CR4. */
1743	VMXDisable();
1744
1745	/* Clear CR4.VMXE only if it was clear prior to use setting it. */
1746	if (!pHostCpu->fVmxeAlreadyEnabled)
1747	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1748
1749	rc = VINF_SUCCESS;
1750	}
1751	else
1752	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1753
1754	/* Restore interrupts. */
1755	ASMSetFlags(fEFlags);
1756	return rc;
1757	}
1758
1759
1760	/**
1761	* Allocates pages specified as specified by an array of VMX page allocation info
1762	* objects.
1763	*
1764	* The pages contents are zero'd after allocation.
1765	*
1766	* @returns VBox status code.
1767	* @param hMemObj The ring-0 memory object associated with the allocation.
1768	* @param paAllocInfo The pointer to the first element of the VMX
1769	* page-allocation info object array.
1770	* @param cEntries The number of elements in the @a paAllocInfo array.
1771	*/
1772	static int hmR0VmxPagesAllocZ(RTR0MEMOBJ hMemObj, PVMXPAGEALLOCINFO paAllocInfo, uint32_t cEntries)
1773	{
1774	/* Figure out how many pages to allocate. */
1775	uint32_t cPages = 0;
1776	for (uint32_t iPage = 0; iPage < cEntries; iPage++)
1777	cPages += !!paAllocInfo[iPage].fValid;
1778
1779	/* Allocate the pages. */
1780	if (cPages)
1781	{
1782	size_t const cbPages = cPages << X86_PAGE_4K_SHIFT;
1783	int rc = RTR0MemObjAllocPage(&hMemObj, cbPages, false /* fExecutable */);
1784	if (RT_FAILURE(rc))
1785	return rc;
1786
1787	/* Zero the contents and assign each page to the corresponding VMX page-allocation entry. */
1788	void *pvFirstPage = RTR0MemObjAddress(hMemObj);
1789	ASMMemZero32(pvFirstPage, cbPages);
1790
1791	uint32_t iPage = 0;
1792	for (uint32_t i = 0; i < cEntries; i++)
1793	if (paAllocInfo[i].fValid)
1794	{
1795	RTHCPHYS const HCPhysPage = RTR0MemObjGetPagePhysAddr(hMemObj, iPage);
1796	void pvPage = (void )((uintptr_t)pvFirstPage + (iPage << X86_PAGE_4K_SHIFT));
1797	Assert(HCPhysPage && HCPhysPage != NIL_RTHCPHYS);
1798	AssertPtr(pvPage);
1799
1800	Assert(paAllocInfo[iPage].pHCPhys);
1801	Assert(paAllocInfo[iPage].ppVirt);
1802	*paAllocInfo[iPage].pHCPhys = HCPhysPage;
1803	*paAllocInfo[iPage].ppVirt = pvPage;
1804
1805	/* Move to next page. */
1806	++iPage;
1807	}
1808
1809	/* Make sure all valid (requested) pages have been assigned. */
1810	Assert(iPage == cPages);
1811	}
1812	return VINF_SUCCESS;
1813	}
1814
1815
1816	/**
1817	* Frees pages allocated using hmR0VmxPagesAllocZ.
1818	*
1819	* @param hMemObj The ring-0 memory object associated with the allocation.
1820	*/
1821	DECL_FORCE_INLINE(void) hmR0VmxPagesFree(RTR0MEMOBJ hMemObj)
1822	{
1823	/* We can cleanup wholesale since it's all one allocation. */
1824	RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
1825	}
1826
1827
1828	/**
1829	* Initializes a VMCS info. object.
1830	*
1831	* @param pVmcsInfo The VMCS info. object.
1832	*/
1833	static void hmR0VmxVmcsInfoInit(PVMXVMCSINFO pVmcsInfo)
1834	{
1835	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1836
1837	Assert(pVmcsInfo->hMemObj == NIL_RTR0MEMOBJ);
1838	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1839	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1840	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1841	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1842	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1843	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1844	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1845	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1846	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1847	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
1848	pVmcsInfo->idHostCpuExec = NIL_RTCPUID;
1849	}
1850
1851
1852	/**
1853	* Frees the VT-x structures for a VMCS info. object.
1854	*
1855	* @param pVmcsInfo The VMCS info. object.
1856	*/
1857	static void hmR0VmxVmcsInfoFree(PVMXVMCSINFO pVmcsInfo)
1858	{
1859	if (pVmcsInfo->hMemObj != NIL_RTR0MEMOBJ)
1860	{
1861	hmR0VmxPagesFree(pVmcsInfo->hMemObj);
1862	hmR0VmxVmcsInfoInit(pVmcsInfo);
1863	}
1864	}
1865
1866
1867	/**
1868	* Allocates the VT-x structures for a VMCS info. object.
1869	*
1870	* @returns VBox status code.
1871	* @param pVCpu The cross context virtual CPU structure.
1872	* @param pVmcsInfo The VMCS info. object.
1873	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1874	*
1875	* @remarks The caller is expected to take care of any and all allocation failures.
1876	* This function will not perform any cleanup for failures half-way
1877	* through.
1878	*/
1879	static int hmR0VmxAllocVmcsInfo(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1880	{
1881	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1882
1883	bool const fMsrBitmaps = RT_BOOL(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS);
1884	bool const fShadowVmcs = !fIsNstGstVmcs ? pVM->hm.s.vmx.fUseVmcsShadowing : pVM->cpum.ro.GuestFeatures.fVmxVmcsShadowing;
1885	Assert(!pVM->cpum.ro.GuestFeatures.fVmxVmcsShadowing); /* VMCS shadowing is not yet exposed to the guest. */
1886	VMXPAGEALLOCINFO aAllocInfo[] = {
1887	{ true, 0 /* Unused */, &pVmcsInfo->HCPhysVmcs, &pVmcsInfo->pvVmcs },
1888	{ true, 0 /* Unused */, &pVmcsInfo->HCPhysGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad },
1889	{ true, 0 /* Unused */, &pVmcsInfo->HCPhysHostMsrLoad, &pVmcsInfo->pvHostMsrLoad },
1890	{ fMsrBitmaps, 0 /* Unused */, &pVmcsInfo->HCPhysMsrBitmap, &pVmcsInfo->pvMsrBitmap },
1891	{ fShadowVmcs, 0 /* Unused */, &pVmcsInfo->HCPhysShadowVmcs, &pVmcsInfo->pvShadowVmcs },
1892	};
1893
1894	int rc = hmR0VmxPagesAllocZ(pVmcsInfo->hMemObj, &aAllocInfo[0], RT_ELEMENTS(aAllocInfo));
1895	if (RT_FAILURE(rc))
1896	return rc;
1897
1898	/*
1899	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1900	* Because they contain a symmetric list of guest MSRs to load on VM-entry and store on VM-exit.
1901	*/
1902	AssertCompile(RT_ELEMENTS(aAllocInfo) > 0);
1903	Assert(pVmcsInfo->HCPhysGuestMsrLoad != NIL_RTHCPHYS);
1904	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1905	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1906
1907	/*
1908	* Get the virtual-APIC page rather than allocating them again.
1909	*/
1910	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1911	{
1912	if (!fIsNstGstVmcs)
1913	{
1914	if (PDMHasApic(pVM))
1915	{
1916	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1917	if (RT_FAILURE(rc))
1918	return rc;
1919	Assert(pVmcsInfo->pbVirtApic);
1920	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1921	}
1922	}
1923	else
1924	{
1925	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(&pVCpu->cpum.GstCtx, &pVmcsInfo->HCPhysVirtApic);
1926	Assert(pVmcsInfo->pbVirtApic);
1927	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1928	}
1929	}
1930
1931	return VINF_SUCCESS;
1932	}
1933
1934
1935	/**
1936	* Free all VT-x structures for the VM.
1937	*
1938	* @returns IPRT status code.
1939	* @param pVM The cross context VM structure.
1940	*/
1941	static void hmR0VmxStructsFree(PVMCC pVM)
1942	{
1943	hmR0VmxPagesFree(pVM->hm.s.vmx.hMemObj);
1944	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1945	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1946	{
1947	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1948	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1949	}
1950	#endif
1951
1952	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1953	{
1954	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1955	hmR0VmxVmcsInfoFree(&pVCpu->hm.s.vmx.VmcsInfo);
1956	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1957	if (pVM->cpum.ro.GuestFeatures.fVmx)
1958	hmR0VmxVmcsInfoFree(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1959	#endif
1960	}
1961	}
1962
1963
1964	/**
1965	* Allocate all VT-x structures for the VM.
1966	*
1967	* @returns IPRT status code.
1968	* @param pVM The cross context VM structure.
1969	*
1970	* @remarks This functions will cleanup on memory allocation failures.
1971	*/
1972	static int hmR0VmxStructsAlloc(PVMCC pVM)
1973	{
1974	/*
1975	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1976	* The VMCS size cannot be more than 4096 bytes.
1977	*
1978	* See Intel spec. Appendix A.1 "Basic VMX Information".
1979	*/
1980	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1981	if (cbVmcs <= X86_PAGE_4K_SIZE)
1982	{ /* likely */ }
1983	else
1984	{
1985	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1986	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1987	}
1988
1989	/*
1990	* Allocate per-VM VT-x structures.
1991	*/
1992	bool const fVirtApicAccess = RT_BOOL(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
1993	bool const fUseVmcsShadowing = pVM->hm.s.vmx.fUseVmcsShadowing;
1994	VMXPAGEALLOCINFO aAllocInfo[] = {
1995	{ fVirtApicAccess, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess },
1996	{ fUseVmcsShadowing, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap },
1997	{ fUseVmcsShadowing, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap },
1998	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1999	{ true, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysScratch, &(PRTR0PTR)pVM->hm.s.vmx.pbScratch },
2000	#endif
2001	};
2002
2003	int rc = hmR0VmxPagesAllocZ(pVM->hm.s.vmx.hMemObj, &aAllocInfo[0], RT_ELEMENTS(aAllocInfo));
2004	if (RT_FAILURE(rc))
2005	goto cleanup;
2006
2007	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2008	/* Allocate the shadow VMCS-fields array. */
2009	if (fUseVmcsShadowing)
2010	{
2011	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
2012	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
2013	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
2014	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
2015	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
2016	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
2017	{ /* likely */ }
2018	else
2019	{
2020	rc = VERR_NO_MEMORY;
2021	goto cleanup;
2022	}
2023	}
2024	#endif
2025
2026	/*
2027	* Allocate per-VCPU VT-x structures.
2028	*/
2029	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2030	{
2031	/* Allocate the guest VMCS structures. */
2032	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
2033	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2034	if (RT_FAILURE(rc))
2035	goto cleanup;
2036
2037	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2038	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2039	if (pVM->cpum.ro.GuestFeatures.fVmx)
2040	{
2041	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2042	if (RT_FAILURE(rc))
2043	goto cleanup;
2044	}
2045	#endif
2046	}
2047
2048	return VINF_SUCCESS;
2049
2050	cleanup:
2051	hmR0VmxStructsFree(pVM);
2052	Assert(rc != VINF_SUCCESS);
2053	return rc;
2054	}
2055
2056
2057	/**
2058	* Pre-initializes non-zero fields in VMX structures that will be allocated.
2059	*
2060	* @param pVM The cross context VM structure.
2061	*/
2062	static void hmR0VmxStructsInit(PVMCC pVM)
2063	{
2064	/* Paranoia. */
2065	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
2066	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2067	Assert(pVM->hm.s.vmx.pbScratch == NULL);
2068	#endif
2069
2070	/*
2071	* Initialize members up-front so we can cleanup en masse on allocation failures.
2072	*/
2073	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2074	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
2075	#endif
2076	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
2077	pVM->hm.s.vmx.HCPhysVmreadBitmap = NIL_RTHCPHYS;
2078	pVM->hm.s.vmx.HCPhysVmwriteBitmap = NIL_RTHCPHYS;
2079	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2080	{
2081	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
2082	hmR0VmxVmcsInfoInit(&pVCpu->hm.s.vmx.VmcsInfo);
2083	hmR0VmxVmcsInfoInit(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
2084	}
2085	}
2086
2087	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2088	/**
2089	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2090	*
2091	* @returns @c true if the MSR is intercepted, @c false otherwise.
2092	* @param pvMsrBitmap The MSR bitmap.
2093	* @param offMsr The MSR byte offset.
2094	* @param iBit The bit offset from the byte offset.
2095	*/
2096	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2097	{
2098	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2099	Assert(pbMsrBitmap);
2100	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2101	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2102	}
2103	#endif
2104
2105	/**
2106	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2107	*
2108	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2109	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2110	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2111	* the read/write access of this MSR.
2112	*
2113	* @param pVCpu The cross context virtual CPU structure.
2114	* @param pVmcsInfo The VMCS info. object.
2115	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2116	* @param idMsr The MSR value.
2117	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2118	* include both a read -and- a write permission!
2119	*
2120	* @sa CPUMGetVmxMsrPermission.
2121	* @remarks Can be called with interrupts disabled.
2122	*/
2123	static void hmR0VmxSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2124	{
2125	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2126	Assert(pbMsrBitmap);
2127	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2128
2129	/*
2130	* MSR-bitmap Layout:
2131	* Byte index MSR range Interpreted as
2132	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2133	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2134	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2135	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2136	*
2137	* A bit corresponding to an MSR within the above range causes a VM-exit
2138	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2139	* the MSR range, it always cause a VM-exit.
2140	*
2141	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2142	*/
2143	uint16_t const offBitmapRead = 0;
2144	uint16_t const offBitmapWrite = 0x800;
2145	uint16_t offMsr;
2146	int32_t iBit;
2147	if (idMsr <= UINT32_C(0x00001fff))
2148	{
2149	offMsr = 0;
2150	iBit = idMsr;
2151	}
2152	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2153	{
2154	offMsr = 0x400;
2155	iBit = idMsr - UINT32_C(0xc0000000);
2156	}
2157	else
2158	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2159
2160	/*
2161	* Set the MSR read permission.
2162	*/
2163	uint16_t const offMsrRead = offBitmapRead + offMsr;
2164	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2165	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2166	{
2167	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2168	bool const fClear = !fIsNstGstVmcs ? true
2169	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2170	#else
2171	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2172	bool const fClear = true;
2173	#endif
2174	if (fClear)
2175	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2176	}
2177	else
2178	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2179
2180	/*
2181	* Set the MSR write permission.
2182	*/
2183	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2184	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2185	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2186	{
2187	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2188	bool const fClear = !fIsNstGstVmcs ? true
2189	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2190	#else
2191	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2192	bool const fClear = true;
2193	#endif
2194	if (fClear)
2195	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2196	}
2197	else
2198	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2199	}
2200
2201
2202	/**
2203	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2204	* area.
2205	*
2206	* @returns VBox status code.
2207	* @param pVCpu The cross context virtual CPU structure.
2208	* @param pVmcsInfo The VMCS info. object.
2209	* @param cMsrs The number of MSRs.
2210	*/
2211	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2212	{
2213	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2214	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2215	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2216	{
2217	/* Commit the MSR counts to the VMCS and update the cache. */
2218	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2219	{
2220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
2222	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2223	pVmcsInfo->cEntryMsrLoad = cMsrs;
2224	pVmcsInfo->cExitMsrStore = cMsrs;
2225	pVmcsInfo->cExitMsrLoad = cMsrs;
2226	}
2227	return VINF_SUCCESS;
2228	}
2229
2230	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2231	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2232	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2233	}
2234
2235
2236	/**
2237	* Adds a new (or updates the value of an existing) guest/host MSR
2238	* pair to be swapped during the world-switch as part of the
2239	* auto-load/store MSR area in the VMCS.
2240	*
2241	* @returns VBox status code.
2242	* @param pVCpu The cross context virtual CPU structure.
2243	* @param pVmxTransient The VMX-transient structure.
2244	* @param idMsr The MSR.
2245	* @param uGuestMsrValue Value of the guest MSR.
2246	* @param fSetReadWrite Whether to set the guest read/write access of this
2247	* MSR (thus not causing a VM-exit).
2248	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2249	* necessary.
2250	*/
2251	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2252	bool fSetReadWrite, bool fUpdateHostMsr)
2253	{
2254	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2255	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2256	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2257	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2258	uint32_t i;
2259
2260	/* Paranoia. */
2261	Assert(pGuestMsrLoad);
2262
2263	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGuestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2264
2265	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2266	for (i = 0; i < cMsrs; i++)
2267	{
2268	if (pGuestMsrLoad[i].u32Msr == idMsr)
2269	break;
2270	}
2271
2272	bool fAdded = false;
2273	if (i == cMsrs)
2274	{
2275	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2276	++cMsrs;
2277	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2278	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2279
2280	/* Set the guest to read/write this MSR without causing VM-exits. */
2281	if ( fSetReadWrite
2282	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2283	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2284
2285	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2286	fAdded = true;
2287	}
2288
2289	/* Update the MSR value for the newly added or already existing MSR. */
2290	pGuestMsrLoad[i].u32Msr = idMsr;
2291	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2292
2293	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2294	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2295	{
2296	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2297	pGuestMsrStore[i].u32Msr = idMsr;
2298	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2299	}
2300
2301	/* Update the corresponding slot in the host MSR area. */
2302	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2303	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2304	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2305	pHostMsr[i].u32Msr = idMsr;
2306
2307	/*
2308	* Only if the caller requests to update the host MSR value AND we've newly added the
2309	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2310	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2311	*
2312	* We do this for performance reasons since reading MSRs may be quite expensive.
2313	*/
2314	if (fAdded)
2315	{
2316	if (fUpdateHostMsr)
2317	{
2318	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2319	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2320	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2321	}
2322	else
2323	{
2324	/* Someone else can do the work. */
2325	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2326	}
2327	}
2328	return VINF_SUCCESS;
2329	}
2330
2331
2332	/**
2333	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2334	* auto-load/store MSR area in the VMCS.
2335	*
2336	* @returns VBox status code.
2337	* @param pVCpu The cross context virtual CPU structure.
2338	* @param pVmxTransient The VMX-transient structure.
2339	* @param idMsr The MSR.
2340	*/
2341	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2342	{
2343	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2344	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2345	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2346	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2347
2348	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2349
2350	for (uint32_t i = 0; i < cMsrs; i++)
2351	{
2352	/* Find the MSR. */
2353	if (pGuestMsrLoad[i].u32Msr == idMsr)
2354	{
2355	/*
2356	* If it's the last MSR, we only need to reduce the MSR count.
2357	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2358	*/
2359	if (i < cMsrs - 1)
2360	{
2361	/* Remove it from the VM-entry MSR-load area. */
2362	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2363	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2364
2365	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2366	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2367	{
2368	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2369	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2370	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2371	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2372	}
2373
2374	/* Remove it from the VM-exit MSR-load area. */
2375	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2376	Assert(pHostMsr[i].u32Msr == idMsr);
2377	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2378	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2379	}
2380
2381	/* Reduce the count to reflect the removed MSR and bail. */
2382	--cMsrs;
2383	break;
2384	}
2385	}
2386
2387	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2388	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2389	{
2390	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2391	AssertRCReturn(rc, rc);
2392
2393	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2394	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2395	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2396
2397	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2398	return VINF_SUCCESS;
2399	}
2400
2401	return VERR_NOT_FOUND;
2402	}
2403
2404
2405	/**
2406	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2407	*
2408	* @returns @c true if found, @c false otherwise.
2409	* @param pVmcsInfo The VMCS info. object.
2410	* @param idMsr The MSR to find.
2411	*/
2412	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2413	{
2414	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2415	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2416	Assert(pMsrs);
2417	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2418	for (uint32_t i = 0; i < cMsrs; i++)
2419	{
2420	if (pMsrs[i].u32Msr == idMsr)
2421	return true;
2422	}
2423	return false;
2424	}
2425
2426
2427	/**
2428	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2429	*
2430	* @param pVCpu The cross context virtual CPU structure.
2431	* @param pVmcsInfo The VMCS info. object.
2432	*
2433	* @remarks No-long-jump zone!!!
2434	*/
2435	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2436	{
2437	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2438
2439	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2440	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2441	Assert(pHostMsrLoad);
2442	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2443	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2444	for (uint32_t i = 0; i < cMsrs; i++)
2445	{
2446	/*
2447	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2448	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2449	*/
2450	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2451	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2452	else
2453	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2454	}
2455	}
2456
2457
2458	/**
2459	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2460	* perform lazy restoration of the host MSRs while leaving VT-x.
2461	*
2462	* @param pVCpu The cross context virtual CPU structure.
2463	*
2464	* @remarks No-long-jump zone!!!
2465	*/
2466	static void hmR0VmxLazySaveHostMsrs(PVMCPUCC pVCpu)
2467	{
2468	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2469
2470	/*
2471	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2472	*/
2473	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2474	{
2475	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2476	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2477	{
2478	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2479	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2480	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2481	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2482	}
2483	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2484	}
2485	}
2486
2487
2488	/**
2489	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2490	* lazily while leaving VT-x.
2491	*
2492	* @returns true if it does, false otherwise.
2493	* @param pVCpu The cross context virtual CPU structure.
2494	* @param idMsr The MSR to check.
2495	*/
2496	static bool hmR0VmxIsLazyGuestMsr(PCVMCPUCC pVCpu, uint32_t idMsr)
2497	{
2498	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2499	{
2500	switch (idMsr)
2501	{
2502	case MSR_K8_LSTAR:
2503	case MSR_K6_STAR:
2504	case MSR_K8_SF_MASK:
2505	case MSR_K8_KERNEL_GS_BASE:
2506	return true;
2507	}
2508	}
2509	return false;
2510	}
2511
2512
2513	/**
2514	* Loads a set of guests MSRs to allow read/passthru to the guest.
2515	*
2516	* The name of this function is slightly confusing. This function does NOT
2517	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2518	* common prefix for functions dealing with "lazy restoration" of the shared
2519	* MSRs.
2520	*
2521	* @param pVCpu The cross context virtual CPU structure.
2522	*
2523	* @remarks No-long-jump zone!!!
2524	*/
2525	static void hmR0VmxLazyLoadGuestMsrs(PVMCPUCC pVCpu)
2526	{
2527	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2528	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2529
2530	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2531	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2532	{
2533	/*
2534	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2535	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2536	* we can skip a few MSR writes.
2537	*
2538	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2539	* guest MSR values in the guest-CPU context might be different to what's currently
2540	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2541	* CPU, see @bugref{8728}.
2542	*/
2543	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2544	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2545	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2546	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2547	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2548	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2549	{
2550	#ifdef VBOX_STRICT
2551	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2552	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2553	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2554	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2555	#endif
2556	}
2557	else
2558	{
2559	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2560	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2561	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2562	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2563	}
2564	}
2565	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2566	}
2567
2568
2569	/**
2570	* Performs lazy restoration of the set of host MSRs if they were previously
2571	* loaded with guest MSR values.
2572	*
2573	* @param pVCpu The cross context virtual CPU structure.
2574	*
2575	* @remarks No-long-jump zone!!!
2576	* @remarks The guest MSRs should have been saved back into the guest-CPU
2577	* context by hmR0VmxImportGuestState()!!!
2578	*/
2579	static void hmR0VmxLazyRestoreHostMsrs(PVMCPUCC pVCpu)
2580	{
2581	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2582	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2583
2584	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2585	{
2586	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2587	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2588	{
2589	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2590	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2591	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2592	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2593	}
2594	}
2595	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2596	}
2597
2598
2599	/**
2600	* Verifies that our cached values of the VMCS fields are all consistent with
2601	* what's actually present in the VMCS.
2602	*
2603	* @returns VBox status code.
2604	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2605	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2606	* VMCS content. HMCPU error-field is
2607	* updated, see VMX_VCI_XXX.
2608	* @param pVCpu The cross context virtual CPU structure.
2609	* @param pVmcsInfo The VMCS info. object.
2610	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2611	*/
2612	static int hmR0VmxCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2613	{
2614	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2615
2616	uint32_t u32Val;
2617	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2618	AssertRC(rc);
2619	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2620	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2621	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2622	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2623
2624	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2625	AssertRC(rc);
2626	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2627	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2628	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2629	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2630
2631	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2632	AssertRC(rc);
2633	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2634	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2635	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2636	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2637
2638	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2639	AssertRC(rc);
2640	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2641	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2642	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2643	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2644
2645	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2646	{
2647	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2648	AssertRC(rc);
2649	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2650	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2651	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2652	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2653	}
2654
2655	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2656	AssertRC(rc);
2657	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2658	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2659	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2660	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2661
2662	uint64_t u64Val;
2663	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2664	AssertRC(rc);
2665	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2666	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2667	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2668	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2669
2670	NOREF(pcszVmcs);
2671	return VINF_SUCCESS;
2672	}
2673
2674
2675	#ifdef VBOX_STRICT
2676	/**
2677	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2678	*
2679	* @param pVCpu The cross context virtual CPU structure.
2680	* @param pVmcsInfo The VMCS info. object.
2681	*/
2682	static void hmR0VmxCheckHostEferMsr(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2683	{
2684	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2685
2686	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2687	{
2688	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2689	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2690	uint64_t uVmcsEferMsrVmcs;
2691	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2692	AssertRC(rc);
2693
2694	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2695	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2696	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2697	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2698	}
2699	}
2700
2701
2702	/**
2703	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2704	* VMCS are correct.
2705	*
2706	* @param pVCpu The cross context virtual CPU structure.
2707	* @param pVmcsInfo The VMCS info. object.
2708	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2709	*/
2710	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2711	{
2712	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2713
2714	/* Read the various MSR-area counts from the VMCS. */
2715	uint32_t cEntryLoadMsrs;
2716	uint32_t cExitStoreMsrs;
2717	uint32_t cExitLoadMsrs;
2718	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2719	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2720	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2721
2722	/* Verify all the MSR counts are the same. */
2723	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2724	Assert(cExitStoreMsrs == cExitLoadMsrs);
2725	uint32_t const cMsrs = cExitLoadMsrs;
2726
2727	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2728	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2729
2730	/* Verify the MSR counts are within the allocated page size. */
2731	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2732
2733	/* Verify the relevant contents of the MSR areas match. */
2734	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2735	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2736	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2737	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2738	for (uint32_t i = 0; i < cMsrs; i++)
2739	{
2740	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2741	if (fSeparateExitMsrStorePage)
2742	{
2743	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2744	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2745	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2746	}
2747
2748	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2749	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2750	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2751
2752	uint64_t const u64HostMsr = ASMRdMsr(pHostMsrLoad->u32Msr);
2753	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64HostMsr,
2754	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2755	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64HostMsr, cMsrs));
2756
2757	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2758	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2759	if (fIsEferMsr)
2760	{
2761	AssertMsgReturnVoid(u64HostMsr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2762	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2763	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64HostMsr, cMsrs));
2764	}
2765
2766	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2767	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2768	{
2769	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2770	if (fIsEferMsr)
2771	{
2772	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2773	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2774	}
2775	else
2776	{
2777	/* Verify LBR MSRs (used only for debugging) are intercepted. We don't passthru these MSRs to the guest yet. */
2778	PCVM pVM = pVCpu->CTX_SUFF(pVM);
2779	if ( pVM->hm.s.vmx.fLbr
2780	&& ( hmR0VmxIsLbrBranchFromMsr(pVM, pGuestMsrLoad->u32Msr, NULL /* pidxMsr */)
2781	\|\| hmR0VmxIsLbrBranchToMsr(pVM, pGuestMsrLoad->u32Msr, NULL /* pidxMsr */)
2782	\|\| pGuestMsrLoad->u32Msr == pVM->hm.s.vmx.idLbrTosMsr))
2783	{
2784	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_MASK) == VMXMSRPM_EXIT_RD_WR,
2785	("u32Msr=%#RX32 cMsrs=%u Passthru read/write for LBR MSRs!\n",
2786	pGuestMsrLoad->u32Msr, cMsrs));
2787	}
2788	else if (!fIsNstGstVmcs)
2789	{
2790	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_MASK) == VMXMSRPM_ALLOW_RD_WR,
2791	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2792	}
2793	else
2794	{
2795	/*
2796	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2797	* execute a nested-guest with MSR passthrough.
2798	*
2799	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2800	* allow passthrough too.
2801	*/
2802	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2803	Assert(pvMsrBitmapNstGst);
2804	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2805	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2806	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2807	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2808	}
2809	}
2810	}
2811
2812	/* Move to the next MSR. */
2813	pHostMsrLoad++;
2814	pGuestMsrLoad++;
2815	pGuestMsrStore++;
2816	}
2817	}
2818	#endif /* VBOX_STRICT */
2819
2820
2821	/**
2822	* Flushes the TLB using EPT.
2823	*
2824	* @returns VBox status code.
2825	* @param pVCpu The cross context virtual CPU structure of the calling
2826	* EMT. Can be NULL depending on @a enmTlbFlush.
2827	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2828	* enmTlbFlush.
2829	* @param enmTlbFlush Type of flush.
2830	*
2831	* @remarks Caller is responsible for making sure this function is called only
2832	* when NestedPaging is supported and providing @a enmTlbFlush that is
2833	* supported by the CPU.
2834	* @remarks Can be called with interrupts disabled.
2835	*/
2836	static void hmR0VmxFlushEpt(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2837	{
2838	uint64_t au64Descriptor[2];
2839	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2840	au64Descriptor[0] = 0;
2841	else
2842	{
2843	Assert(pVCpu);
2844	Assert(pVmcsInfo);
2845	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2846	}
2847	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2848
2849	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2850	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2851
2852	if ( RT_SUCCESS(rc)
2853	&& pVCpu)
2854	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2855	}
2856
2857
2858	/**
2859	* Flushes the TLB using VPID.
2860	*
2861	* @returns VBox status code.
2862	* @param pVCpu The cross context virtual CPU structure of the calling
2863	* EMT. Can be NULL depending on @a enmTlbFlush.
2864	* @param enmTlbFlush Type of flush.
2865	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2866	* on @a enmTlbFlush).
2867	*
2868	* @remarks Can be called with interrupts disabled.
2869	*/
2870	static void hmR0VmxFlushVpid(PVMCPUCC pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2871	{
2872	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2873
2874	uint64_t au64Descriptor[2];
2875	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2876	{
2877	au64Descriptor[0] = 0;
2878	au64Descriptor[1] = 0;
2879	}
2880	else
2881	{
2882	AssertPtr(pVCpu);
2883	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2884	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2885	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2886	au64Descriptor[1] = GCPtr;
2887	}
2888
2889	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2890	AssertMsg(rc == VINF_SUCCESS,
2891	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2892
2893	if ( RT_SUCCESS(rc)
2894	&& pVCpu)
2895	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2896	NOREF(rc);
2897	}
2898
2899
2900	/**
2901	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2902	* otherwise there is nothing really to invalidate.
2903	*
2904	* @returns VBox status code.
2905	* @param pVCpu The cross context virtual CPU structure.
2906	* @param GCVirt Guest virtual address of the page to invalidate.
2907	*/
2908	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
2909	{
2910	AssertPtr(pVCpu);
2911	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2912
2913	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2914	{
2915	/*
2916	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2917	* the EPT case. See @bugref{6043} and @bugref{6177}.
2918	*
2919	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2920	* as this function maybe called in a loop with individual addresses.
2921	*/
2922	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2923	if (pVM->hm.s.vmx.fVpid)
2924	{
2925	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2926	if (fVpidFlush)
2927	{
2928	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2929	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2930	}
2931	else
2932	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2933	}
2934	else if (pVM->hm.s.fNestedPaging)
2935	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2936	}
2937
2938	return VINF_SUCCESS;
2939	}
2940
2941
2942	/**
2943	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2944	* case where neither EPT nor VPID is supported by the CPU.
2945	*
2946	* @param pHostCpu The HM physical-CPU structure.
2947	* @param pVCpu The cross context virtual CPU structure.
2948	*
2949	* @remarks Called with interrupts disabled.
2950	*/
2951	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2952	{
2953	AssertPtr(pVCpu);
2954	AssertPtr(pHostCpu);
2955
2956	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2957
2958	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2959	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2960	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2961	pVCpu->hm.s.fForceTLBFlush = false;
2962	return;
2963	}
2964
2965
2966	/**
2967	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2968	*
2969	* @param pHostCpu The HM physical-CPU structure.
2970	* @param pVCpu The cross context virtual CPU structure.
2971	* @param pVmcsInfo The VMCS info. object.
2972	*
2973	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2974	* nomenclature. The reason is, to avoid confusion in compare statements
2975	* since the host-CPU copies are named "ASID".
2976	*
2977	* @remarks Called with interrupts disabled.
2978	*/
2979	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2980	{
2981	#ifdef VBOX_WITH_STATISTICS
2982	bool fTlbFlushed = false;
2983	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2984	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2985	if (!fTlbFlushed) \
2986	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2987	} while (0)
2988	#else
2989	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2990	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2991	#endif
2992
2993	AssertPtr(pVCpu);
2994	AssertPtr(pHostCpu);
2995	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2996
2997	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2998	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2999	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
3000	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
3001
3002	/*
3003	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
3004	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3005	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3006	* cannot reuse the current ASID anymore.
3007	*/
3008	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3009	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3010	{
3011	++pHostCpu->uCurrentAsid;
3012	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3013	{
3014	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
3015	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3016	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3017	}
3018
3019	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3020	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3021	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3022
3023	/*
3024	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
3025	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
3026	*/
3027	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3028	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3029	HMVMX_SET_TAGGED_TLB_FLUSHED();
3030	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
3031	}
3032	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
3033	{
3034	/*
3035	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
3036	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
3037	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
3038	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
3039	* mappings, see @bugref{6568}.
3040	*
3041	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
3042	*/
3043	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3044	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3045	HMVMX_SET_TAGGED_TLB_FLUSHED();
3046	}
3047	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3048	{
3049	/*
3050	* The nested-guest specifies its own guest-physical address to use as the APIC-access
3051	* address which requires flushing the TLB of EPT cached structures.
3052	*
3053	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
3054	*/
3055	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3056	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3057	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3058	HMVMX_SET_TAGGED_TLB_FLUSHED();
3059	}
3060
3061
3062	pVCpu->hm.s.fForceTLBFlush = false;
3063	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3064
3065	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3066	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3067	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3068	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3069	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3070	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3071	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3072	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3073	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3074
3075	/* Update VMCS with the VPID. */
3076	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3077	AssertRC(rc);
3078
3079	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3080	}
3081
3082
3083	/**
3084	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3085	*
3086	* @param pHostCpu The HM physical-CPU structure.
3087	* @param pVCpu The cross context virtual CPU structure.
3088	* @param pVmcsInfo The VMCS info. object.
3089	*
3090	* @remarks Called with interrupts disabled.
3091	*/
3092	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3093	{
3094	AssertPtr(pVCpu);
3095	AssertPtr(pHostCpu);
3096	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3097	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3098	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3099
3100	/*
3101	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3102	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3103	*/
3104	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3105	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3106	{
3107	pVCpu->hm.s.fForceTLBFlush = true;
3108	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3109	}
3110
3111	/* Check for explicit TLB flushes. */
3112	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3113	{
3114	pVCpu->hm.s.fForceTLBFlush = true;
3115	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3116	}
3117
3118	/* Check for TLB flushes while switching to/from a nested-guest. */
3119	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3120	{
3121	pVCpu->hm.s.fForceTLBFlush = true;
3122	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3123	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3124	}
3125
3126	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3127	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3128
3129	if (pVCpu->hm.s.fForceTLBFlush)
3130	{
3131	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3132	pVCpu->hm.s.fForceTLBFlush = false;
3133	}
3134	}
3135
3136
3137	/**
3138	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3139	*
3140	* @param pHostCpu The HM physical-CPU structure.
3141	* @param pVCpu The cross context virtual CPU structure.
3142	*
3143	* @remarks Called with interrupts disabled.
3144	*/
3145	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
3146	{
3147	AssertPtr(pVCpu);
3148	AssertPtr(pHostCpu);
3149	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3150	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3151	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3152
3153	/*
3154	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3155	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3156	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3157	* cannot reuse the current ASID anymore.
3158	*/
3159	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3160	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3161	{
3162	pVCpu->hm.s.fForceTLBFlush = true;
3163	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3164	}
3165
3166	/* Check for explicit TLB flushes. */
3167	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3168	{
3169	/*
3170	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3171	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3172	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3173	* include fExplicitFlush's too) - an obscure corner case.
3174	*/
3175	pVCpu->hm.s.fForceTLBFlush = true;
3176	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3177	}
3178
3179	/* Check for TLB flushes while switching to/from a nested-guest. */
3180	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3181	{
3182	pVCpu->hm.s.fForceTLBFlush = true;
3183	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3184	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3185	}
3186
3187	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3188	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3189	if (pVCpu->hm.s.fForceTLBFlush)
3190	{
3191	++pHostCpu->uCurrentAsid;
3192	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3193	{
3194	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3195	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3196	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3197	}
3198
3199	pVCpu->hm.s.fForceTLBFlush = false;
3200	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3201	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3202	if (pHostCpu->fFlushAsidBeforeUse)
3203	{
3204	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3205	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3206	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3207	{
3208	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3209	pHostCpu->fFlushAsidBeforeUse = false;
3210	}
3211	else
3212	{
3213	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3214	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3215	}
3216	}
3217	}
3218
3219	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3220	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3221	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3222	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3223	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3224	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3225	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3226
3227	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3228	AssertRC(rc);
3229	}
3230
3231
3232	/**
3233	* Flushes the guest TLB entry based on CPU capabilities.
3234	*
3235	* @param pHostCpu The HM physical-CPU structure.
3236	* @param pVCpu The cross context virtual CPU structure.
3237	* @param pVmcsInfo The VMCS info. object.
3238	*
3239	* @remarks Called with interrupts disabled.
3240	*/
3241	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3242	{
3243	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3244	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3245	#endif
3246	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3247	switch (pVM->hm.s.vmx.enmTlbFlushType)
3248	{
3249	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3250	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3251	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3252	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3253	default:
3254	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3255	break;
3256	}
3257	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3258	}
3259
3260
3261	/**
3262	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3263	* TLB entries from the host TLB before VM-entry.
3264	*
3265	* @returns VBox status code.
3266	* @param pVM The cross context VM structure.
3267	*/
3268	static int hmR0VmxSetupTaggedTlb(PVMCC pVM)
3269	{
3270	/*
3271	* Determine optimal flush type for nested paging.
3272	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3273	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3274	*/
3275	if (pVM->hm.s.fNestedPaging)
3276	{
3277	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3278	{
3279	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3280	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3281	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3282	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3283	else
3284	{
3285	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3286	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3287	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3288	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3289	}
3290
3291	/* Make sure the write-back cacheable memory type for EPT is supported. */
3292	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3293	{
3294	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3295	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3296	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3297	}
3298
3299	/* EPT requires a page-walk length of 4. */
3300	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3301	{
3302	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3303	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3304	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3305	}
3306	}
3307	else
3308	{
3309	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3310	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3311	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3312	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3313	}
3314	}
3315
3316	/*
3317	* Determine optimal flush type for VPID.
3318	*/
3319	if (pVM->hm.s.vmx.fVpid)
3320	{
3321	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3322	{
3323	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3324	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3325	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3326	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3327	else
3328	{
3329	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3330	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3331	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3332	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3333	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3334	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3335	pVM->hm.s.vmx.fVpid = false;
3336	}
3337	}
3338	else
3339	{
3340	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3341	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3342	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3343	pVM->hm.s.vmx.fVpid = false;
3344	}
3345	}
3346
3347	/*
3348	* Setup the handler for flushing tagged-TLBs.
3349	*/
3350	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3351	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3352	else if (pVM->hm.s.fNestedPaging)
3353	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3354	else if (pVM->hm.s.vmx.fVpid)
3355	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3356	else
3357	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3358	return VINF_SUCCESS;
3359	}
3360
3361
3362	/**
3363	* Sets up the LBR MSR ranges based on the host CPU.
3364	*
3365	* @returns VBox status code.
3366	* @param pVM The cross context VM structure.
3367	*/
3368	static int hmR0VmxSetupLbrMsrRange(PVMCC pVM)
3369	{
3370	Assert(pVM->hm.s.vmx.fLbr);
3371	uint32_t idLbrFromIpMsrFirst;
3372	uint32_t idLbrFromIpMsrLast;
3373	uint32_t idLbrToIpMsrFirst;
3374	uint32_t idLbrToIpMsrLast;
3375	uint32_t idLbrTosMsr;
3376
3377	/*
3378	* Determine the LBR MSRs supported for this host CPU family and model.
3379	*
3380	* See Intel spec. 17.4.8 "LBR Stack".
3381	* See Intel "Model-Specific Registers" spec.
3382	*/
3383	uint32_t const uFamilyModel = (pVM->cpum.ro.HostFeatures.uFamily << 8)
3384	\| pVM->cpum.ro.HostFeatures.uModel;
3385	switch (uFamilyModel)
3386	{
3387	case 0x0f01: case 0x0f02:
3388	idLbrFromIpMsrFirst = MSR_P4_LASTBRANCH_0;
3389	idLbrFromIpMsrLast = MSR_P4_LASTBRANCH_3;
3390	idLbrToIpMsrFirst = 0x0;
3391	idLbrToIpMsrLast = 0x0;
3392	idLbrTosMsr = MSR_P4_LASTBRANCH_TOS;
3393	break;
3394
3395	case 0x065c: case 0x065f: case 0x064e: case 0x065e: case 0x068e:
3396	case 0x069e: case 0x0655: case 0x0666: case 0x067a: case 0x0667:
3397	case 0x066a: case 0x066c: case 0x067d: case 0x067e:
3398	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
3399	idLbrFromIpMsrLast = MSR_LASTBRANCH_31_FROM_IP;
3400	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
3401	idLbrToIpMsrLast = MSR_LASTBRANCH_31_TO_IP;
3402	idLbrTosMsr = MSR_LASTBRANCH_TOS;
3403	break;
3404
3405	case 0x063d: case 0x0647: case 0x064f: case 0x0656: case 0x063c:
3406	case 0x0645: case 0x0646: case 0x063f: case 0x062a: case 0x062d:
3407	case 0x063a: case 0x063e: case 0x061a: case 0x061e: case 0x061f:
3408	case 0x062e: case 0x0625: case 0x062c: case 0x062f:
3409	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
3410	idLbrFromIpMsrLast = MSR_LASTBRANCH_15_FROM_IP;
3411	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
3412	idLbrToIpMsrLast = MSR_LASTBRANCH_15_TO_IP;
3413	idLbrTosMsr = MSR_LASTBRANCH_TOS;
3414	break;
3415
3416	case 0x0617: case 0x061d: case 0x060f:
3417	idLbrFromIpMsrFirst = MSR_CORE2_LASTBRANCH_0_FROM_IP;
3418	idLbrFromIpMsrLast = MSR_CORE2_LASTBRANCH_3_FROM_IP;
3419	idLbrToIpMsrFirst = MSR_CORE2_LASTBRANCH_0_TO_IP;
3420	idLbrToIpMsrLast = MSR_CORE2_LASTBRANCH_3_TO_IP;
3421	idLbrTosMsr = MSR_CORE2_LASTBRANCH_TOS;
3422	break;
3423
3424	/* Atom and related microarchitectures we don't care about:
3425	case 0x0637: case 0x064a: case 0x064c: case 0x064d: case 0x065a:
3426	case 0x065d: case 0x061c: case 0x0626: case 0x0627: case 0x0635:
3427	case 0x0636: */
3428	/* All other CPUs: */
3429	default:
3430	{
3431	LogRelFunc(("Could not determine LBR stack size for the CPU model %#x\n", uFamilyModel));
3432	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_UNKNOWN;
3433	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3434	}
3435	}
3436
3437	/*
3438	* Validate.
3439	*/
3440	uint32_t const cLbrStack = idLbrFromIpMsrLast - idLbrFromIpMsrFirst + 1;
3441	PCVMCPU pVCpu0 = VMCC_GET_CPU_0(pVM);
3442	AssertCompile( RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr)
3443	== RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrToIpMsr));
3444	if (cLbrStack > RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr))
3445	{
3446	LogRelFunc(("LBR stack size of the CPU (%u) exceeds our buffer size\n", cLbrStack));
3447	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_OVERFLOW;
3448	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3449	}
3450	NOREF(pVCpu0);
3451
3452	/*
3453	* Update the LBR info. to the VM struct. for use later.
3454	*/
3455	pVM->hm.s.vmx.idLbrTosMsr = idLbrTosMsr;
3456	pVM->hm.s.vmx.idLbrFromIpMsrFirst = idLbrFromIpMsrFirst;
3457	pVM->hm.s.vmx.idLbrFromIpMsrLast = idLbrFromIpMsrLast;
3458
3459	pVM->hm.s.vmx.idLbrToIpMsrFirst = idLbrToIpMsrFirst;
3460	pVM->hm.s.vmx.idLbrToIpMsrLast = idLbrToIpMsrLast;
3461	return VINF_SUCCESS;
3462	}
3463
3464
3465	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3466	/**
3467	* Sets up the shadow VMCS fields arrays.
3468	*
3469	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3470	* executing the guest.
3471	*
3472	* @returns VBox status code.
3473	* @param pVM The cross context VM structure.
3474	*/
3475	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVMCC pVM)
3476	{
3477	/*
3478	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3479	* when the host does not support it.
3480	*/
3481	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3482	if ( !fGstVmwriteAll
3483	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3484	{ /* likely. */ }
3485	else
3486	{
3487	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3488	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3489	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3490	}
3491
3492	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3493	uint32_t cRwFields = 0;
3494	uint32_t cRoFields = 0;
3495	for (uint32_t i = 0; i < cVmcsFields; i++)
3496	{
3497	VMXVMCSFIELD VmcsField;
3498	VmcsField.u = g_aVmcsFields[i];
3499
3500	/*
3501	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3502	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3503	* in the shadow VMCS fields array as they would be redundant.
3504	*
3505	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3506	* we must not include it in the shadow VMCS fields array. Guests attempting to
3507	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3508	* the required behavior.
3509	*/
3510	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3511	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3512	{
3513	/*
3514	* Read-only fields are placed in a separate array so that while syncing shadow
3515	* VMCS fields later (which is more performance critical) we can avoid branches.
3516	*
3517	* However, if the guest can write to all fields (including read-only fields),
3518	* we treat it a as read/write field. Otherwise, writing to these fields would
3519	* cause a VMWRITE instruction error while syncing the shadow VMCS.
3520	*/
3521	if ( fGstVmwriteAll
3522	\|\| !VMXIsVmcsFieldReadOnly(VmcsField.u))
3523	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3524	else
3525	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3526	}
3527	}
3528
3529	/* Update the counts. */
3530	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3531	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3532	return VINF_SUCCESS;
3533	}
3534
3535
3536	/**
3537	* Sets up the VMREAD and VMWRITE bitmaps.
3538	*
3539	* @param pVM The cross context VM structure.
3540	*/
3541	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVMCC pVM)
3542	{
3543	/*
3544	* By default, ensure guest attempts to access any VMCS fields cause VM-exits.
3545	*/
3546	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3547	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3548	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3549	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3550	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3551
3552	/*
3553	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3554	* VMREAD and VMWRITE bitmaps.
3555	*/
3556	{
3557	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3558	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3559	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3560	{
3561	uint32_t const uVmcsField = paShadowVmcsFields[i];
3562	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3563	Assert(uVmcsField >> 3 < cbBitmap);
3564	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3565	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3566	}
3567	}
3568
3569	/*
3570	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3571	* if the host supports VMWRITE to all supported VMCS fields.
3572	*/
3573	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3574	{
3575	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3576	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3577	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3578	{
3579	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3580	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3581	Assert(uVmcsField >> 3 < cbBitmap);
3582	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3583	}
3584	}
3585	}
3586	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3587
3588
3589	/**
3590	* Sets up the virtual-APIC page address for the VMCS.
3591	*
3592	* @param pVmcsInfo The VMCS info. object.
3593	*/
3594	DECLINLINE(void) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
3595	{
3596	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3597	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3598	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3599	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3600	AssertRC(rc);
3601	}
3602
3603
3604	/**
3605	* Sets up the MSR-bitmap address for the VMCS.
3606	*
3607	* @param pVmcsInfo The VMCS info. object.
3608	*/
3609	DECLINLINE(void) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
3610	{
3611	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3612	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3613	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3614	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3615	AssertRC(rc);
3616	}
3617
3618
3619	/**
3620	* Sets up the APIC-access page address for the VMCS.
3621	*
3622	* @param pVCpu The cross context virtual CPU structure.
3623	*/
3624	DECLINLINE(void) hmR0VmxSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
3625	{
3626	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3627	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3628	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3629	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3630	AssertRC(rc);
3631	}
3632
3633
3634	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3635	/**
3636	* Sets up the VMREAD bitmap address for the VMCS.
3637	*
3638	* @param pVCpu The cross context virtual CPU structure.
3639	*/
3640	DECLINLINE(void) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
3641	{
3642	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3643	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3644	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3645	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3646	AssertRC(rc);
3647	}
3648
3649
3650	/**
3651	* Sets up the VMWRITE bitmap address for the VMCS.
3652	*
3653	* @param pVCpu The cross context virtual CPU structure.
3654	*/
3655	DECLINLINE(void) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
3656	{
3657	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3658	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3659	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3660	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3661	AssertRC(rc);
3662	}
3663	#endif
3664
3665
3666	/**
3667	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3668	* in the VMCS.
3669	*
3670	* @returns VBox status code.
3671	* @param pVmcsInfo The VMCS info. object.
3672	*/
3673	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
3674	{
3675	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3676	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3677	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3678
3679	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3680	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3681	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3682
3683	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3684	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3685	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3686
3687	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad); AssertRC(rc);
3688	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore); AssertRC(rc);
3689	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad); AssertRC(rc);
3690	return VINF_SUCCESS;
3691	}
3692
3693
3694	/**
3695	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3696	*
3697	* @param pVCpu The cross context virtual CPU structure.
3698	* @param pVmcsInfo The VMCS info. object.
3699	*/
3700	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3701	{
3702	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3703
3704	/*
3705	* By default, ensure guest attempts to access any MSR cause VM-exits.
3706	* This shall later be relaxed for specific MSRs as necessary.
3707	*
3708	* Note: For nested-guests, the entire bitmap will be merged prior to
3709	* executing the nested-guest using hardware-assisted VMX and hence there
3710	* is no need to perform this operation. See hmR0VmxMergeMsrBitmapNested.
3711	*/
3712	Assert(pVmcsInfo->pvMsrBitmap);
3713	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
3714
3715	/*
3716	* The guest can access the following MSRs (read, write) without causing
3717	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3718	*/
3719	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3720	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3721	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3722	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3723	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3724	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3725
3726	/*
3727	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3728	* associated with then. We never need to intercept access (writes need to be
3729	* executed without causing a VM-exit, reads will #GP fault anyway).
3730	*
3731	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3732	* read/write them. We swap the the guest/host MSR value using the
3733	* auto-load/store MSR area.
3734	*/
3735	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3736	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3737	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3738	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3739	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3740	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3741
3742	/*
3743	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3744	* required for 64-bit guests.
3745	*/
3746	if (pVM->hm.s.fAllow64BitGuests)
3747	{
3748	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3749	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3750	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3751	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3752	}
3753
3754	/*
3755	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3756	*/
3757	#ifdef VBOX_STRICT
3758	Assert(pVmcsInfo->pvMsrBitmap);
3759	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3760	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3761	#endif
3762	}
3763
3764
3765	/**
3766	* Sets up pin-based VM-execution controls in the VMCS.
3767	*
3768	* @returns VBox status code.
3769	* @param pVCpu The cross context virtual CPU structure.
3770	* @param pVmcsInfo The VMCS info. object.
3771	*/
3772	static int hmR0VmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3773	{
3774	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3775	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3776	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3777
3778	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3779	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3780
3781	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3782	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3783
3784	/* Enable the VMX-preemption timer. */
3785	if (pVM->hm.s.vmx.fUsePreemptTimer)
3786	{
3787	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3788	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3789	}
3790
3791	#if 0
3792	/* Enable posted-interrupt processing. */
3793	if (pVM->hm.s.fPostedIntrs)
3794	{
3795	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3796	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3797	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3798	}
3799	#endif
3800
3801	if ((fVal & fZap) != fVal)
3802	{
3803	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3804	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3805	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3806	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3807	}
3808
3809	/* Commit it to the VMCS and update our cache. */
3810	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3811	AssertRC(rc);
3812	pVmcsInfo->u32PinCtls = fVal;
3813
3814	return VINF_SUCCESS;
3815	}
3816
3817
3818	/**
3819	* Sets up secondary processor-based VM-execution controls in the VMCS.
3820	*
3821	* @returns VBox status code.
3822	* @param pVCpu The cross context virtual CPU structure.
3823	* @param pVmcsInfo The VMCS info. object.
3824	*/
3825	static int hmR0VmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3826	{
3827	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3828	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3829	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3830
3831	/* WBINVD causes a VM-exit. */
3832	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3833	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3834
3835	/* Enable EPT (aka nested-paging). */
3836	if (pVM->hm.s.fNestedPaging)
3837	fVal \|= VMX_PROC_CTLS2_EPT;
3838
3839	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3840	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3841	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3842	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3843	fVal \|= VMX_PROC_CTLS2_INVPCID;
3844
3845	/* Enable VPID. */
3846	if (pVM->hm.s.vmx.fVpid)
3847	fVal \|= VMX_PROC_CTLS2_VPID;
3848
3849	/* Enable unrestricted guest execution. */
3850	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3851	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3852
3853	#if 0
3854	if (pVM->hm.s.fVirtApicRegs)
3855	{
3856	/* Enable APIC-register virtualization. */
3857	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3858	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3859
3860	/* Enable virtual-interrupt delivery. */
3861	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3862	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3863	}
3864	#endif
3865
3866	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3867	where the TPR shadow resides. */
3868	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3869	* done dynamically. */
3870	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3871	{
3872	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3873	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3874	}
3875
3876	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3877	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3878	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3879	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3880	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3881
3882	/* Enable Pause-Loop exiting. */
3883	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3884	&& pVM->hm.s.vmx.cPleGapTicks
3885	&& pVM->hm.s.vmx.cPleWindowTicks)
3886	{
3887	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3888
3889	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
3890	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
3891	}
3892
3893	if ((fVal & fZap) != fVal)
3894	{
3895	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3896	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3897	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3898	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3899	}
3900
3901	/* Commit it to the VMCS and update our cache. */
3902	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3903	AssertRC(rc);
3904	pVmcsInfo->u32ProcCtls2 = fVal;
3905
3906	return VINF_SUCCESS;
3907	}
3908
3909
3910	/**
3911	* Sets up processor-based VM-execution controls in the VMCS.
3912	*
3913	* @returns VBox status code.
3914	* @param pVCpu The cross context virtual CPU structure.
3915	* @param pVmcsInfo The VMCS info. object.
3916	*/
3917	static int hmR0VmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3918	{
3919	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3920	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3921	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3922
3923	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3924	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3925	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3926	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3927	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3928	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3929	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3930
3931	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3932	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3933	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3934	{
3935	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3936	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3937	}
3938
3939	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3940	if (!pVM->hm.s.fNestedPaging)
3941	{
3942	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3943	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3944	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3945	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3946	}
3947
3948	/* Use TPR shadowing if supported by the CPU. */
3949	if ( PDMHasApic(pVM)
3950	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
3951	{
3952	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3953	/* CR8 writes cause a VM-exit based on TPR threshold. */
3954	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3955	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3956	hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3957	}
3958	else
3959	{
3960	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3961	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3962	if (pVM->hm.s.fAllow64BitGuests)
3963	{
3964	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3965	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3966	}
3967	}
3968
3969	/* Use MSR-bitmaps if supported by the CPU. */
3970	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3971	{
3972	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3973	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3974	}
3975
3976	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3977	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3978	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3979
3980	if ((fVal & fZap) != fVal)
3981	{
3982	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3983	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3984	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3985	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3986	}
3987
3988	/* Commit it to the VMCS and update our cache. */
3989	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3990	AssertRC(rc);
3991	pVmcsInfo->u32ProcCtls = fVal;
3992
3993	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3994	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3995	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3996
3997	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3998	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3999	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
4000
4001	/* Sanity check, should not really happen. */
4002	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
4003	{ /* likely */ }
4004	else
4005	{
4006	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
4007	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4008	}
4009
4010	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
4011	return VINF_SUCCESS;
4012	}
4013
4014
4015	/**
4016	* Sets up miscellaneous (everything other than Pin, Processor and secondary
4017	* Processor-based VM-execution) control fields in the VMCS.
4018	*
4019	* @returns VBox status code.
4020	* @param pVCpu The cross context virtual CPU structure.
4021	* @param pVmcsInfo The VMCS info. object.
4022	*/
4023	static int hmR0VmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4024	{
4025	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4026	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
4027	{
4028	hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
4029	hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
4030	}
4031	#endif
4032
4033	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
4034	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
4035	AssertRC(rc);
4036
4037	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
4038	if (RT_SUCCESS(rc))
4039	{
4040	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
4041	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
4042
4043	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
4044	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
4045
4046	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
4047	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
4048
4049	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fLbr)
4050	{
4051	rc = VMXWriteVmcsNw(VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
4052	AssertRC(rc);
4053	}
4054	return VINF_SUCCESS;
4055	}
4056	else
4057	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
4058	return rc;
4059	}
4060
4061
4062	/**
4063	* Sets up the initial exception bitmap in the VMCS based on static conditions.
4064	*
4065	* We shall setup those exception intercepts that don't change during the
4066	* lifetime of the VM here. The rest are done dynamically while loading the
4067	* guest state.
4068	*
4069	* @param pVCpu The cross context virtual CPU structure.
4070	* @param pVmcsInfo The VMCS info. object.
4071	*/
4072	static void hmR0VmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4073	{
4074	/*
4075	* The following exceptions are always intercepted:
4076	*
4077	* #AC - To prevent the guest from hanging the CPU.
4078	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
4079	* recursive #DBs can cause a CPU hang.
4080	* #PF - To sync our shadow page tables when nested-paging is not used.
4081	*/
4082	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
4083	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
4084	\| RT_BIT(X86_XCPT_DB)
4085	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
4086
4087	/* Commit it to the VMCS. */
4088	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4089	AssertRC(rc);
4090
4091	/* Update our cache of the exception bitmap. */
4092	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4093	}
4094
4095
4096	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4097	/**
4098	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
4099	*
4100	* @returns VBox status code.
4101	* @param pVCpu The cross context virtual CPU structure.
4102	* @param pVmcsInfo The VMCS info. object.
4103	*/
4104	static int hmR0VmxSetupVmcsCtlsNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4105	{
4106	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
4107	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
4108	AssertRC(rc);
4109
4110	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
4111	if (RT_SUCCESS(rc))
4112	{
4113	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4114	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
4115
4116	/* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
4117	Assert(!pVmcsInfo->u64Cr0Mask);
4118	Assert(!pVmcsInfo->u64Cr4Mask);
4119	return VINF_SUCCESS;
4120	}
4121	else
4122	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
4123	return rc;
4124	}
4125	#endif
4126
4127
4128	/**
4129	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
4130	* VMX.
4131	*
4132	* @returns VBox status code.
4133	* @param pVCpu The cross context virtual CPU structure.
4134	* @param pVmcsInfo The VMCS info. object.
4135	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
4136	*/
4137	static int hmR0VmxSetupVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
4138	{
4139	Assert(pVmcsInfo->pvVmcs);
4140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4141
4142	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
4143	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4144	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4145	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
4146
4147	LogFlowFunc(("\n"));
4148
4149	/*
4150	* Initialize the VMCS using VMCLEAR before loading the VMCS.
4151	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
4152	*/
4153	int rc = hmR0VmxClearVmcs(pVmcsInfo);
4154	if (RT_SUCCESS(rc))
4155	{
4156	rc = hmR0VmxLoadVmcs(pVmcsInfo);
4157	if (RT_SUCCESS(rc))
4158	{
4159	/*
4160	* Initialize the hardware-assisted VMX execution handler for guest and nested-guest VMCS.
4161	* The host is always 64-bit since we no longer support 32-bit hosts.
4162	* Currently we have just a single handler for all guest modes as well, see @bugref{6208#c73}.
4163	*/
4164	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
4165	if (!fIsNstGstVmcs)
4166	{
4167	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
4168	if (RT_SUCCESS(rc))
4169	{
4170	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4171	if (RT_SUCCESS(rc))
4172	{
4173	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4174	if (RT_SUCCESS(rc))
4175	{
4176	hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4177	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4178	/*
4179	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4180	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4181	* making it fit for use when VMCS shadowing is later enabled.
4182	*/
4183	if (pVmcsInfo->pvShadowVmcs)
4184	{
4185	VMXVMCSREVID VmcsRevId;
4186	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4187	VmcsRevId.n.fIsShadowVmcs = 1;
4188	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4189	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4190	if (RT_SUCCESS(rc))
4191	{ /* likely */ }
4192	else
4193	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4194	}
4195	#endif
4196	}
4197	else
4198	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4199	}
4200	else
4201	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4202	}
4203	else
4204	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4205	}
4206	else
4207	{
4208	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4209	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4210	if (RT_SUCCESS(rc))
4211	{ /* likely */ }
4212	else
4213	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4214	#else
4215	AssertFailed();
4216	#endif
4217	}
4218	}
4219	else
4220	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4221	}
4222	else
4223	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4224
4225	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4226	if (RT_SUCCESS(rc))
4227	{
4228	rc = hmR0VmxClearVmcs(pVmcsInfo);
4229	if (RT_SUCCESS(rc))
4230	{ /* likely */ }
4231	else
4232	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4233	}
4234
4235	/*
4236	* Update the last-error record both for failures and success, so we
4237	* can propagate the status code back to ring-3 for diagnostics.
4238	*/
4239	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4240	NOREF(pszVmcs);
4241	return rc;
4242	}
4243
4244
4245	/**
4246	* Does global VT-x initialization (called during module initialization).
4247	*
4248	* @returns VBox status code.
4249	*/
4250	VMMR0DECL(int) VMXR0GlobalInit(void)
4251	{
4252	#ifdef HMVMX_USE_FUNCTION_TABLE
4253	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_aVMExitHandlers));
4254	# ifdef VBOX_STRICT
4255	for (unsigned i = 0; i < RT_ELEMENTS(g_aVMExitHandlers); i++)
4256	Assert(g_aVMExitHandlers[i].pfn);
4257	# endif
4258	#endif
4259	return VINF_SUCCESS;
4260	}
4261
4262
4263	/**
4264	* Does global VT-x termination (called during module termination).
4265	*/
4266	VMMR0DECL(void) VMXR0GlobalTerm()
4267	{
4268	/* Nothing to do currently. */
4269	}
4270
4271
4272	/**
4273	* Sets up and activates VT-x on the current CPU.
4274	*
4275	* @returns VBox status code.
4276	* @param pHostCpu The HM physical-CPU structure.
4277	* @param pVM The cross context VM structure. Can be
4278	* NULL after a host resume operation.
4279	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4280	* fEnabledByHost is @c true).
4281	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4282	* @a fEnabledByHost is @c true).
4283	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4284	* enable VT-x on the host.
4285	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4286	*/
4287	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4288	PCSUPHWVIRTMSRS pHwvirtMsrs)
4289	{
4290	AssertPtr(pHostCpu);
4291	AssertPtr(pHwvirtMsrs);
4292	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4293
4294	/* Enable VT-x if it's not already enabled by the host. */
4295	if (!fEnabledByHost)
4296	{
4297	int rc = hmR0VmxEnterRootMode(pHostCpu, pVM, HCPhysCpuPage, pvCpuPage);
4298	if (RT_FAILURE(rc))
4299	return rc;
4300	}
4301
4302	/*
4303	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4304	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4305	* invalidated when flushing by VPID.
4306	*/
4307	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4308	{
4309	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4310	pHostCpu->fFlushAsidBeforeUse = false;
4311	}
4312	else
4313	pHostCpu->fFlushAsidBeforeUse = true;
4314
4315	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4316	++pHostCpu->cTlbFlushes;
4317
4318	return VINF_SUCCESS;
4319	}
4320
4321
4322	/**
4323	* Deactivates VT-x on the current CPU.
4324	*
4325	* @returns VBox status code.
4326	* @param pHostCpu The HM physical-CPU structure.
4327	* @param pvCpuPage Pointer to the VMXON region.
4328	* @param HCPhysCpuPage Physical address of the VMXON region.
4329	*
4330	* @remarks This function should never be called when SUPR0EnableVTx() or
4331	* similar was used to enable VT-x on the host.
4332	*/
4333	VMMR0DECL(int) VMXR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4334	{
4335	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4336
4337	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4338	return hmR0VmxLeaveRootMode(pHostCpu);
4339	}
4340
4341
4342	/**
4343	* Does per-VM VT-x initialization.
4344	*
4345	* @returns VBox status code.
4346	* @param pVM The cross context VM structure.
4347	*/
4348	VMMR0DECL(int) VMXR0InitVM(PVMCC pVM)
4349	{
4350	AssertPtr(pVM);
4351	LogFlowFunc(("pVM=%p\n", pVM));
4352
4353	hmR0VmxStructsInit(pVM);
4354	int rc = hmR0VmxStructsAlloc(pVM);
4355	if (RT_FAILURE(rc))
4356	{
4357	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4358	return rc;
4359	}
4360
4361	/* Setup the crash dump page. */
4362	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4363	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
4364	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
4365	#endif
4366	return VINF_SUCCESS;
4367	}
4368
4369
4370	/**
4371	* Does per-VM VT-x termination.
4372	*
4373	* @returns VBox status code.
4374	* @param pVM The cross context VM structure.
4375	*/
4376	VMMR0DECL(int) VMXR0TermVM(PVMCC pVM)
4377	{
4378	AssertPtr(pVM);
4379	LogFlowFunc(("pVM=%p\n", pVM));
4380
4381	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4382	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4383	{
4384	Assert(pVM->hm.s.vmx.pvScratch);
4385	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4386	}
4387	#endif
4388	hmR0VmxStructsFree(pVM);
4389	return VINF_SUCCESS;
4390	}
4391
4392
4393	/**
4394	* Sets up the VM for execution using hardware-assisted VMX.
4395	* This function is only called once per-VM during initialization.
4396	*
4397	* @returns VBox status code.
4398	* @param pVM The cross context VM structure.
4399	*/
4400	VMMR0DECL(int) VMXR0SetupVM(PVMCC pVM)
4401	{
4402	AssertPtr(pVM);
4403	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4404
4405	LogFlowFunc(("pVM=%p\n", pVM));
4406
4407	/*
4408	* At least verify if VMX is enabled, since we can't check if we're in VMX root mode or not
4409	* without causing a #GP.
4410	*/
4411	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4412	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4413	{ /* likely */ }
4414	else
4415	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4416
4417	/*
4418	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4419	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4420	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4421	*/
4422	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4423	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4424	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4425	{
4426	LogRelFunc(("Invalid real-on-v86 state.\n"));
4427	return VERR_INTERNAL_ERROR;
4428	}
4429
4430	/* Initialize these always, see hmR3InitFinalizeR0().*/
4431	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4432	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4433
4434	/* Setup the tagged-TLB flush handlers. */
4435	int rc = hmR0VmxSetupTaggedTlb(pVM);
4436	if (RT_FAILURE(rc))
4437	{
4438	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4439	return rc;
4440	}
4441
4442	/* Determine LBR capabilities. */
4443	if (pVM->hm.s.vmx.fLbr)
4444	{
4445	rc = hmR0VmxSetupLbrMsrRange(pVM);
4446	if (RT_FAILURE(rc))
4447	{
4448	LogRelFunc(("Failed to setup LBR MSR range. rc=%Rrc\n", rc));
4449	return rc;
4450	}
4451	}
4452
4453	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4454	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4455	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4456	{
4457	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4458	if (RT_SUCCESS(rc))
4459	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4460	else
4461	{
4462	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4463	return rc;
4464	}
4465	}
4466	#endif
4467
4468	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4469	{
4470	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
4471	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4472
4473	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4474	if (RT_SUCCESS(rc))
4475	{
4476	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4477	if (pVM->cpum.ro.GuestFeatures.fVmx)
4478	{
4479	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4480	if (RT_SUCCESS(rc))
4481	{ /* likely */ }
4482	else
4483	{
4484	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4485	return rc;
4486	}
4487	}
4488	#endif
4489	}
4490	else
4491	{
4492	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4493	return rc;
4494	}
4495	}
4496
4497	return VINF_SUCCESS;
4498	}
4499
4500
4501	/**
4502	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4503	* the VMCS.
4504	*/
4505	static void hmR0VmxExportHostControlRegs(void)
4506	{
4507	int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, ASMGetCR0()); AssertRC(rc);
4508	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, ASMGetCR3()); AssertRC(rc);
4509	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, ASMGetCR4()); AssertRC(rc);
4510	}
4511
4512
4513	/**
4514	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4515	* the host-state area in the VMCS.
4516	*
4517	* @returns VBox status code.
4518	* @param pVCpu The cross context virtual CPU structure.
4519	*/
4520	static int hmR0VmxExportHostSegmentRegs(PVMCPUCC pVCpu)
4521	{
4522	/**
4523	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4524	* requirements. See hmR0VmxExportHostSegmentRegs().
4525	*/
4526	#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_selValue) \
4527	if ((a_selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4528	{ \
4529	bool fValidSelector = true; \
4530	if ((a_selValue) & X86_SEL_LDT) \
4531	{ \
4532	uint32_t const uAttr = ASMGetSegAttr(a_selValue); \
4533	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4534	} \
4535	if (fValidSelector) \
4536	{ \
4537	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##a_Seg; \
4538	pVCpu->hm.s.vmx.RestoreHost.uHostSel##a_Seg = (a_selValue); \
4539	} \
4540	(a_selValue) = 0; \
4541	}
4542
4543	/*
4544	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4545	* will be messed up. We should -not- save the messed up state without restoring
4546	* the original host-state, see @bugref{7240}.
4547	*
4548	* This apparently can happen (most likely the FPU changes), deal with it rather than
4549	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4550	*/
4551	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4552	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4553	{
4554	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4555	pVCpu->idCpu));
4556	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4557	}
4558	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4559
4560	/*
4561	* Host segment registers.
4562	*/
4563	RTSEL uSelES = ASMGetES();
4564	RTSEL uSelCS = ASMGetCS();
4565	RTSEL uSelSS = ASMGetSS();
4566	RTSEL uSelDS = ASMGetDS();
4567	RTSEL uSelFS = ASMGetFS();
4568	RTSEL uSelGS = ASMGetGS();
4569	RTSEL uSelTR = ASMGetTR();
4570
4571	/*
4572	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4573	* gain VM-entry and restore them before we get preempted.
4574	*
4575	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4576	*/
4577	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4578	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4579	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4580	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4581
4582	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4583	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4584	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4585	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4586	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4587	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4588	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4589	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4590	Assert(uSelCS);
4591	Assert(uSelTR);
4592
4593	/* Write these host selector fields into the host-state area in the VMCS. */
4594	int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS); AssertRC(rc);
4595	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS); AssertRC(rc);
4596	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS); AssertRC(rc);
4597	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES); AssertRC(rc);
4598	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS); AssertRC(rc);
4599	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS); AssertRC(rc);
4600	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR); AssertRC(rc);
4601
4602	/*
4603	* Host GDTR and IDTR.
4604	*/
4605	RTGDTR Gdtr;
4606	RTIDTR Idtr;
4607	RT_ZERO(Gdtr);
4608	RT_ZERO(Idtr);
4609	ASMGetGDTR(&Gdtr);
4610	ASMGetIDTR(&Idtr);
4611	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRC(rc);
4612	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRC(rc);
4613
4614	/*
4615	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4616	* them to the maximum limit (0xffff) on every VM-exit.
4617	*/
4618	if (Gdtr.cbGdt != 0xffff)
4619	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4620
4621	/*
4622	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4623	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4624	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4625	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4626	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4627	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4628	* at 0xffff on hosts where we are sure it won't cause trouble.
4629	*/
4630	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4631	if (Idtr.cbIdt < 0x0fff)
4632	#else
4633	if (Idtr.cbIdt != 0xffff)
4634	#endif
4635	{
4636	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4637	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4638	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4639	}
4640
4641	/*
4642	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4643	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4644	* RPL should be too in most cases.
4645	*/
4646	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4647	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4648
4649	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4650	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4651
4652	/*
4653	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4654	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4655	* restoration if the host has something else. Task switching is not supported in 64-bit
4656	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4657	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4658	*
4659	* [1] See Intel spec. 3.5 "System Descriptor Types".
4660	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4661	*/
4662	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4663	Assert(pDesc->System.u4Type == 11);
4664	if ( pDesc->System.u16LimitLow != 0x67
4665	\|\| pDesc->System.u4LimitHigh)
4666	{
4667	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4668	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4669	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4670	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4671	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4672	}
4673
4674	/*
4675	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4676	*/
4677	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4678	{
4679	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4680	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4681	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4682	{
4683	/* The GDT is read-only but the writable GDT is available. */
4684	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4685	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4686	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4687	AssertRCReturn(rc, rc);
4688	}
4689	}
4690
4691	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase);
4692	AssertRC(rc);
4693
4694	/*
4695	* Host FS base and GS base.
4696	*/
4697	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4698	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4699	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRC(rc);
4700	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRC(rc);
4701
4702	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4703	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4704	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4705	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4706	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4707
4708	return VINF_SUCCESS;
4709	#undef VMXLOCAL_ADJUST_HOST_SEG
4710	}
4711
4712
4713	/**
4714	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4715	* host-state area of the VMCS.
4716	*
4717	* These MSRs will be automatically restored on the host after every successful
4718	* VM-exit.
4719	*
4720	* @param pVCpu The cross context virtual CPU structure.
4721	*
4722	* @remarks No-long-jump zone!!!
4723	*/
4724	static void hmR0VmxExportHostMsrs(PVMCPUCC pVCpu)
4725	{
4726	AssertPtr(pVCpu);
4727
4728	/*
4729	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4730	* rather than swapping them on every VM-entry.
4731	*/
4732	hmR0VmxLazySaveHostMsrs(pVCpu);
4733
4734	/*
4735	* Host Sysenter MSRs.
4736	*/
4737	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)); AssertRC(rc);
4738	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP)); AssertRC(rc);
4739	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP)); AssertRC(rc);
4740
4741	/*
4742	* Host EFER MSR.
4743	*
4744	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4745	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4746	*/
4747	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4748	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4749	{
4750	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4751	AssertRC(rc);
4752	}
4753
4754	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4755	* hmR0VmxExportGuestEntryExitCtls(). */
4756	}
4757
4758
4759	/**
4760	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4761	*
4762	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4763	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4764	*
4765	* @returns true if we need to load guest EFER, false otherwise.
4766	* @param pVCpu The cross context virtual CPU structure.
4767	* @param pVmxTransient The VMX-transient structure.
4768	*
4769	* @remarks Requires EFER, CR4.
4770	* @remarks No-long-jump zone!!!
4771	*/
4772	static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4773	{
4774	#ifdef HMVMX_ALWAYS_SWAP_EFER
4775	RT_NOREF2(pVCpu, pVmxTransient);
4776	return true;
4777	#else
4778	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4779	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4780	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4781	uint64_t const u64GuestEfer = pCtx->msrEFER;
4782
4783	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4784	/*
4785	* For nested-guests, we shall honor swapping the EFER MSR when requested by
4786	* the nested-guest.
4787	*/
4788	if ( pVmxTransient->fIsNestedGuest
4789	&& ( CPUMIsGuestVmxEntryCtlsSet(pCtx, VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4790	\|\| CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_SAVE_EFER_MSR)
4791	\|\| CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_LOAD_EFER_MSR)))
4792	return true;
4793	# else
4794	RT_NOREF(pVmxTransient);
4795	#endif
4796
4797	/*
4798	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4799	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4800	*/
4801	if ( CPUMIsGuestInLongModeEx(pCtx)
4802	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4803	return true;
4804
4805	/*
4806	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4807	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4808	*
4809	* See Intel spec. 4.5 "IA-32e Paging".
4810	* See Intel spec. 4.1.1 "Three Paging Modes".
4811	*
4812	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4813	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4814	*/
4815	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4816	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4817	if ( (pCtx->cr4 & X86_CR4_PAE)
4818	&& (pCtx->cr0 & X86_CR0_PG))
4819	{
4820	/*
4821	* If nested paging is not used, verify that the guest paging mode matches the
4822	* shadow paging mode which is/will be placed in the VMCS (which is what will
4823	* actually be used while executing the guest and not the CR4 shadow value).
4824	*/
4825	AssertMsg(pVM->hm.s.fNestedPaging \|\| ( pVCpu->hm.s.enmShadowMode == PGMMODE_PAE
4826	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_PAE_NX
4827	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64
4828	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64_NX),
4829	("enmShadowMode=%u\n", pVCpu->hm.s.enmShadowMode));
4830	if ((u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4831	{
4832	/* Verify that the host is NX capable. */
4833	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4834	return true;
4835	}
4836	}
4837
4838	return false;
4839	#endif
4840	}
4841
4842
4843	/**
4844	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4845	* VMCS.
4846	*
4847	* This is typically required when the guest changes paging mode.
4848	*
4849	* @returns VBox status code.
4850	* @param pVCpu The cross context virtual CPU structure.
4851	* @param pVmxTransient The VMX-transient structure.
4852	*
4853	* @remarks Requires EFER.
4854	* @remarks No-long-jump zone!!!
4855	*/
4856	static int hmR0VmxExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4857	{
4858	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4859	{
4860	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4861	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4862
4863	/*
4864	* VM-entry controls.
4865	*/
4866	{
4867	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4868	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4869
4870	/*
4871	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4872	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4873	*
4874	* For nested-guests, this is a mandatory VM-entry control. It's also
4875	* required because we do not want to leak host bits to the nested-guest.
4876	*/
4877	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4878
4879	/*
4880	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4881	*
4882	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4883	* required to get the nested-guest working with hardware-assisted VMX execution.
4884	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
4885	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4886	* here rather than while merging the guest VMCS controls.
4887	*/
4888	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4889	{
4890	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
4891	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4892	}
4893	else
4894	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4895
4896	/*
4897	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4898	*
4899	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4900	* regardless of whether the nested-guest VMCS specifies it because we are free to
4901	* load whatever MSRs we require and we do not need to modify the guest visible copy
4902	* of the VM-entry MSR load area.
4903	*/
4904	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4905	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4906	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4907	else
4908	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4909
4910	/*
4911	* The following should -not- be set (since we're not in SMM mode):
4912	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4913	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4914	*/
4915
4916	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4917	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4918
4919	if ((fVal & fZap) == fVal)
4920	{ /* likely */ }
4921	else
4922	{
4923	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4924	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4925	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4926	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4927	}
4928
4929	/* Commit it to the VMCS. */
4930	if (pVmcsInfo->u32EntryCtls != fVal)
4931	{
4932	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4933	AssertRC(rc);
4934	pVmcsInfo->u32EntryCtls = fVal;
4935	}
4936	}
4937
4938	/*
4939	* VM-exit controls.
4940	*/
4941	{
4942	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4943	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4944
4945	/*
4946	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4947	* supported the 1-setting of this bit.
4948	*
4949	* For nested-guests, we set the "save debug controls" as the converse
4950	* "load debug controls" is mandatory for nested-guests anyway.
4951	*/
4952	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4953
4954	/*
4955	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4956	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4957	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4958	* hmR0VmxExportHostMsrs().
4959	*
4960	* For nested-guests, we always set this bit as we do not support 32-bit
4961	* hosts.
4962	*/
4963	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4964
4965	/*
4966	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4967	*
4968	* For nested-guests, we should use the "save IA32_EFER" control if we also
4969	* used the "load IA32_EFER" control while exporting VM-entry controls.
4970	*/
4971	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4972	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4973	{
4974	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4975	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4976	}
4977
4978	/*
4979	* Enable saving of the VMX-preemption timer value on VM-exit.
4980	* For nested-guests, currently not exposed/used.
4981	*/
4982	if ( pVM->hm.s.vmx.fUsePreemptTimer
4983	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4984	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4985
4986	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4987	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4988
4989	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4990	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4991	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4992
4993	if ((fVal & fZap) == fVal)
4994	{ /* likely */ }
4995	else
4996	{
4997	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4998	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4999	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
5000	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5001	}
5002
5003	/* Commit it to the VMCS. */
5004	if (pVmcsInfo->u32ExitCtls != fVal)
5005	{
5006	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5007	AssertRC(rc);
5008	pVmcsInfo->u32ExitCtls = fVal;
5009	}
5010	}
5011
5012	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5013	}
5014	return VINF_SUCCESS;
5015	}
5016
5017
5018	/**
5019	* Sets the TPR threshold in the VMCS.
5020	*
5021	* @param pVmcsInfo The VMCS info. object.
5022	* @param u32TprThreshold The TPR threshold (task-priority class only).
5023	*/
5024	DECLINLINE(void) hmR0VmxApicSetTprThreshold(PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5025	{
5026	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5027	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5028	RT_NOREF(pVmcsInfo);
5029	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5030	AssertRC(rc);
5031	}
5032
5033
5034	/**
5035	* Exports the guest APIC TPR state into the VMCS.
5036	*
5037	* @param pVCpu The cross context virtual CPU structure.
5038	* @param pVmxTransient The VMX-transient structure.
5039	*
5040	* @remarks No-long-jump zone!!!
5041	*/
5042	static void hmR0VmxExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5043	{
5044	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5045	{
5046	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5047
5048	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5049	if (!pVmxTransient->fIsNestedGuest)
5050	{
5051	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5052	&& APICIsEnabled(pVCpu))
5053	{
5054	/*
5055	* Setup TPR shadowing.
5056	*/
5057	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5058	{
5059	bool fPendingIntr = false;
5060	uint8_t u8Tpr = 0;
5061	uint8_t u8PendingIntr = 0;
5062	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5063	AssertRC(rc);
5064
5065	/*
5066	* If there are interrupts pending but masked by the TPR, instruct VT-x to
5067	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5068	* priority of the pending interrupt so we can deliver the interrupt. If there
5069	* are no interrupts pending, set threshold to 0 to not cause any
5070	* TPR-below-threshold VM-exits.
5071	*/
5072	uint32_t u32TprThreshold = 0;
5073	if (fPendingIntr)
5074	{
5075	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5076	(which is the Task-Priority Class). */
5077	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5078	const uint8_t u8TprPriority = u8Tpr >> 4;
5079	if (u8PendingPriority <= u8TprPriority)
5080	u32TprThreshold = u8PendingPriority;
5081	}
5082
5083	hmR0VmxApicSetTprThreshold(pVmcsInfo, u32TprThreshold);
5084	}
5085	}
5086	}
5087	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5088	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5089	}
5090	}
5091
5092
5093	/**
5094	* Gets the guest interruptibility-state and updates related force-flags.
5095	*
5096	* @returns Guest's interruptibility-state.
5097	* @param pVCpu The cross context virtual CPU structure.
5098	*
5099	* @remarks No-long-jump zone!!!
5100	*/
5101	static uint32_t hmR0VmxGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu)
5102	{
5103	/*
5104	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5105	*/
5106	uint32_t fIntrState = 0;
5107	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5108	{
5109	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
5110	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
5111
5112	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5113	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5114	{
5115	if (pCtx->eflags.Bits.u1IF)
5116	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5117	else
5118	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5119	}
5120	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5121	{
5122	/*
5123	* We can clear the inhibit force flag as even if we go back to the recompiler
5124	* without executing guest code in VT-x, the flag's condition to be cleared is
5125	* met and thus the cleared state is correct.
5126	*/
5127	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5128	}
5129	}
5130
5131	/*
5132	* Check if we should inhibit NMI delivery.
5133	*/
5134	if (CPUMIsGuestNmiBlocking(pVCpu))
5135	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5136
5137	/*
5138	* Validate.
5139	*/
5140	#ifdef VBOX_STRICT
5141	/* We don't support block-by-SMI yet.*/
5142	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
5143
5144	/* Block-by-STI must not be set when interrupts are disabled. */
5145	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5146	{
5147	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5148	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
5149	}
5150	#endif
5151
5152	return fIntrState;
5153	}
5154
5155
5156	/**
5157	* Exports the exception intercepts required for guest execution in the VMCS.
5158	*
5159	* @param pVCpu The cross context virtual CPU structure.
5160	* @param pVmxTransient The VMX-transient structure.
5161	*
5162	* @remarks No-long-jump zone!!!
5163	*/
5164	static void hmR0VmxExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5165	{
5166	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5167	{
5168	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5169	if ( !pVmxTransient->fIsNestedGuest
5170	&& pVCpu->hm.s.fGIMTrapXcptUD)
5171	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5172	else
5173	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5174
5175	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5176	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5177	}
5178	}
5179
5180
5181	/**
5182	* Exports the guest's RIP into the guest-state area in the VMCS.
5183	*
5184	* @param pVCpu The cross context virtual CPU structure.
5185	*
5186	* @remarks No-long-jump zone!!!
5187	*/
5188	static void hmR0VmxExportGuestRip(PVMCPUCC pVCpu)
5189	{
5190	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5191	{
5192	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5193
5194	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5195	AssertRC(rc);
5196
5197	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5198	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5199	}
5200	}
5201
5202
5203	/**
5204	* Exports the guest's RSP into the guest-state area in the VMCS.
5205	*
5206	* @param pVCpu The cross context virtual CPU structure.
5207	*
5208	* @remarks No-long-jump zone!!!
5209	*/
5210	static void hmR0VmxExportGuestRsp(PVMCPUCC pVCpu)
5211	{
5212	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5213	{
5214	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5215
5216	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5217	AssertRC(rc);
5218
5219	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5220	Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
5221	}
5222	}
5223
5224
5225	/**
5226	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5227	*
5228	* @param pVCpu The cross context virtual CPU structure.
5229	* @param pVmxTransient The VMX-transient structure.
5230	*
5231	* @remarks No-long-jump zone!!!
5232	*/
5233	static void hmR0VmxExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5234	{
5235	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5236	{
5237	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5238
5239	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5240	Let us assert it as such and use 32-bit VMWRITE. */
5241	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5242	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5243	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5244	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5245
5246	/*
5247	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5248	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5249	* can run the real-mode guest code under Virtual 8086 mode.
5250	*/
5251	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5252	if (pVmcsInfo->RealMode.fRealOnV86Active)
5253	{
5254	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5255	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5256	Assert(!pVmxTransient->fIsNestedGuest);
5257	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5258	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5259	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5260	}
5261
5262	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5263	AssertRC(rc);
5264
5265	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5266	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5267	}
5268	}
5269
5270
5271	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5272	/**
5273	* Copies the nested-guest VMCS to the shadow VMCS.
5274	*
5275	* @returns VBox status code.
5276	* @param pVCpu The cross context virtual CPU structure.
5277	* @param pVmcsInfo The VMCS info. object.
5278	*
5279	* @remarks No-long-jump zone!!!
5280	*/
5281	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5282	{
5283	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5284	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5285
5286	/*
5287	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5288	* current VMCS, as we may try saving guest lazy MSRs.
5289	*
5290	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5291	* calling the import VMCS code which is currently performing the guest MSR reads
5292	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5293	* and the rest of the VMX leave session machinery.
5294	*/
5295	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5296
5297	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5298	if (RT_SUCCESS(rc))
5299	{
5300	/*
5301	* Copy all guest read/write VMCS fields.
5302	*
5303	* We don't check for VMWRITE failures here for performance reasons and
5304	* because they are not expected to fail, barring irrecoverable conditions
5305	* like hardware errors.
5306	*/
5307	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5308	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5309	{
5310	uint64_t u64Val;
5311	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5312	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5313	VMXWriteVmcs64(uVmcsField, u64Val);
5314	}
5315
5316	/*
5317	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5318	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5319	*/
5320	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5321	{
5322	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5323	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5324	{
5325	uint64_t u64Val;
5326	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5327	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5328	VMXWriteVmcs64(uVmcsField, u64Val);
5329	}
5330	}
5331
5332	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5333	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5334	}
5335
5336	ASMSetFlags(fEFlags);
5337	return rc;
5338	}
5339
5340
5341	/**
5342	* Copies the shadow VMCS to the nested-guest VMCS.
5343	*
5344	* @returns VBox status code.
5345	* @param pVCpu The cross context virtual CPU structure.
5346	* @param pVmcsInfo The VMCS info. object.
5347	*
5348	* @remarks Called with interrupts disabled.
5349	*/
5350	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5351	{
5352	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5353	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5354	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5355
5356	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5357	if (RT_SUCCESS(rc))
5358	{
5359	/*
5360	* Copy guest read/write fields from the shadow VMCS.
5361	* Guest read-only fields cannot be modified, so no need to copy them.
5362	*
5363	* We don't check for VMREAD failures here for performance reasons and
5364	* because they are not expected to fail, barring irrecoverable conditions
5365	* like hardware errors.
5366	*/
5367	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5368	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5369	{
5370	uint64_t u64Val;
5371	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5372	VMXReadVmcs64(uVmcsField, &u64Val);
5373	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5374	}
5375
5376	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5377	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5378	}
5379	return rc;
5380	}
5381
5382
5383	/**
5384	* Enables VMCS shadowing for the given VMCS info. object.
5385	*
5386	* @param pVmcsInfo The VMCS info. object.
5387	*
5388	* @remarks No-long-jump zone!!!
5389	*/
5390	static void hmR0VmxEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5391	{
5392	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5393	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5394	{
5395	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5396	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5397	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5398	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
5399	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5400	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5401	Log4Func(("Enabled\n"));
5402	}
5403	}
5404
5405
5406	/**
5407	* Disables VMCS shadowing for the given VMCS info. object.
5408	*
5409	* @param pVmcsInfo The VMCS info. object.
5410	*
5411	* @remarks No-long-jump zone!!!
5412	*/
5413	static void hmR0VmxDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5414	{
5415	/*
5416	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5417	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5418	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5419	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5420	*
5421	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5422	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5423	*/
5424	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5425	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5426	{
5427	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5428	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5429	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
5430	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5431	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5432	Log4Func(("Disabled\n"));
5433	}
5434	}
5435	#endif
5436
5437
5438	/**
5439	* Exports the guest hardware-virtualization state.
5440	*
5441	* @returns VBox status code.
5442	* @param pVCpu The cross context virtual CPU structure.
5443	* @param pVmxTransient The VMX-transient structure.
5444	*
5445	* @remarks No-long-jump zone!!!
5446	*/
5447	static int hmR0VmxExportGuestHwvirtState(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5448	{
5449	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5450	{
5451	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5452	/*
5453	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5454	* VMCS shadowing.
5455	*/
5456	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5457	{
5458	/*
5459	* If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
5460	* copy the nested hypervisor's current VMCS into the shadow VMCS and enable
5461	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5462	*
5463	* We check for VMX root mode here in case the guest executes VMXOFF without
5464	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5465	* not clear the current VMCS pointer.
5466	*/
5467	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5468	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5469	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5470	&& CPUMIsGuestVmxCurrentVmcsValid(&pVCpu->cpum.GstCtx))
5471	{
5472	/* Paranoia. */
5473	Assert(!pVmxTransient->fIsNestedGuest);
5474
5475	/*
5476	* For performance reasons, also check if the nested hypervisor's current VMCS
5477	* was newly loaded or modified before copying it to the shadow VMCS.
5478	*/
5479	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5480	{
5481	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5482	AssertRCReturn(rc, rc);
5483	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5484	}
5485	hmR0VmxEnableVmcsShadowing(pVmcsInfo);
5486	}
5487	else
5488	hmR0VmxDisableVmcsShadowing(pVmcsInfo);
5489	}
5490	#else
5491	NOREF(pVmxTransient);
5492	#endif
5493	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5494	}
5495	return VINF_SUCCESS;
5496	}
5497
5498
5499	/**
5500	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5501	*
5502	* The guest FPU state is always pre-loaded hence we don't need to bother about
5503	* sharing FPU related CR0 bits between the guest and host.
5504	*
5505	* @returns VBox status code.
5506	* @param pVCpu The cross context virtual CPU structure.
5507	* @param pVmxTransient The VMX-transient structure.
5508	*
5509	* @remarks No-long-jump zone!!!
5510	*/
5511	static int hmR0VmxExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5512	{
5513	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5514	{
5515	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5516	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5517
5518	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0;
5519	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5520	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5521	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5522	else
5523	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5524
5525	if (!pVmxTransient->fIsNestedGuest)
5526	{
5527	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5528	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5529	uint64_t const u64ShadowCr0 = u64GuestCr0;
5530	Assert(!RT_HI_U32(u64GuestCr0));
5531
5532	/*
5533	* Setup VT-x's view of the guest CR0.
5534	*/
5535	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5536	if (pVM->hm.s.fNestedPaging)
5537	{
5538	if (CPUMIsGuestPagingEnabled(pVCpu))
5539	{
5540	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5541	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5542	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5543	}
5544	else
5545	{
5546	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5547	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5548	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5549	}
5550
5551	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5552	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5553	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5554	}
5555	else
5556	{
5557	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5558	u64GuestCr0 \|= X86_CR0_WP;
5559	}
5560
5561	/*
5562	* Guest FPU bits.
5563	*
5564	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5565	* using CR0.TS.
5566	*
5567	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5568	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5569	*/
5570	u64GuestCr0 \|= X86_CR0_NE;
5571
5572	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5573	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5574
5575	/*
5576	* Update exception intercepts.
5577	*/
5578	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5579	if (pVmcsInfo->RealMode.fRealOnV86Active)
5580	{
5581	Assert(PDMVmmDevHeapIsEnabled(pVM));
5582	Assert(pVM->hm.s.vmx.pRealModeTSS);
5583	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5584	}
5585	else
5586	{
5587	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5588	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5589	if (fInterceptMF)
5590	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5591	}
5592
5593	/* Additional intercepts for debugging, define these yourself explicitly. */
5594	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5595	uXcptBitmap \|= 0
5596	\| RT_BIT(X86_XCPT_BP)
5597	\| RT_BIT(X86_XCPT_DE)
5598	\| RT_BIT(X86_XCPT_NM)
5599	\| RT_BIT(X86_XCPT_TS)
5600	\| RT_BIT(X86_XCPT_UD)
5601	\| RT_BIT(X86_XCPT_NP)
5602	\| RT_BIT(X86_XCPT_SS)
5603	\| RT_BIT(X86_XCPT_GP)
5604	\| RT_BIT(X86_XCPT_PF)
5605	\| RT_BIT(X86_XCPT_MF)
5606	;
5607	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5608	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5609	#endif
5610	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5611	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5612	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5613
5614	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5615	u64GuestCr0 \|= fSetCr0;
5616	u64GuestCr0 &= fZapCr0;
5617	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5618
5619	/* Commit the CR0 and related fields to the guest VMCS. */
5620	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5621	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5622	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5623	{
5624	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5625	AssertRC(rc);
5626	}
5627	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5628	{
5629	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5630	AssertRC(rc);
5631	}
5632
5633	/* Update our caches. */
5634	pVmcsInfo->u32ProcCtls = uProcCtls;
5635	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5636
5637	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5638	}
5639	else
5640	{
5641	/*
5642	* With nested-guests, we may have extended the guest/host mask here since we
5643	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5644	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
5645	* originally supplied. We must copy those bits from the nested-guest CR0 into
5646	* the nested-guest CR0 read-shadow.
5647	*/
5648	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5649	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5650	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5651	Assert(!RT_HI_U32(u64GuestCr0));
5652	Assert(u64GuestCr0 & X86_CR0_NE);
5653
5654	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5655	u64GuestCr0 \|= fSetCr0;
5656	u64GuestCr0 &= fZapCr0;
5657	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5658
5659	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5660	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5661	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5662
5663	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5664	}
5665
5666	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5667	}
5668
5669	return VINF_SUCCESS;
5670	}
5671
5672
5673	/**
5674	* Exports the guest control registers (CR3, CR4) into the guest-state area
5675	* in the VMCS.
5676	*
5677	* @returns VBox strict status code.
5678	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5679	* without unrestricted guest access and the VMMDev is not presently
5680	* mapped (e.g. EFI32).
5681	*
5682	* @param pVCpu The cross context virtual CPU structure.
5683	* @param pVmxTransient The VMX-transient structure.
5684	*
5685	* @remarks No-long-jump zone!!!
5686	*/
5687	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5688	{
5689	int rc = VINF_SUCCESS;
5690	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5691
5692	/*
5693	* Guest CR2.
5694	* It's always loaded in the assembler code. Nothing to do here.
5695	*/
5696
5697	/*
5698	* Guest CR3.
5699	*/
5700	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5701	{
5702	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5703
5704	if (pVM->hm.s.fNestedPaging)
5705	{
5706	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5707	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5708
5709	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5710	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5711	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5712	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5713
5714	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5715	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5716	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5717
5718	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5719	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5720	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5721	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5722	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5723	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5724	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5725
5726	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5727	AssertRC(rc);
5728
5729	uint64_t u64GuestCr3;
5730	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5731	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5732	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5733	{
5734	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5735	if (CPUMIsGuestInPAEModeEx(pCtx))
5736	{
5737	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5738	AssertRC(rc);
5739	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
5740	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
5741	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
5742	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
5743	}
5744
5745	/*
5746	* The guest's view of its CR3 is unblemished with nested paging when the
5747	* guest is using paging or we have unrestricted guest execution to handle
5748	* the guest when it's not using paging.
5749	*/
5750	u64GuestCr3 = pCtx->cr3;
5751	}
5752	else
5753	{
5754	/*
5755	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5756	* thinks it accesses physical memory directly, we use our identity-mapped
5757	* page table to map guest-linear to guest-physical addresses. EPT takes care
5758	* of translating it to host-physical addresses.
5759	*/
5760	RTGCPHYS GCPhys;
5761	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5762
5763	/* We obtain it here every time as the guest could have relocated this PCI region. */
5764	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5765	if (RT_SUCCESS(rc))
5766	{ /* likely */ }
5767	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5768	{
5769	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5770	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5771	}
5772	else
5773	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5774
5775	u64GuestCr3 = GCPhys;
5776	}
5777
5778	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
5779	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, u64GuestCr3);
5780	AssertRC(rc);
5781	}
5782	else
5783	{
5784	Assert(!pVmxTransient->fIsNestedGuest);
5785	/* Non-nested paging case, just use the hypervisor's CR3. */
5786	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5787
5788	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
5789	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5790	AssertRC(rc);
5791	}
5792
5793	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5794	}
5795
5796	/*
5797	* Guest CR4.
5798	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5799	*/
5800	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5801	{
5802	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5803	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5804
5805	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0;
5806	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5807
5808	/*
5809	* With nested-guests, we may have extended the guest/host mask here (since we
5810	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5811	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5812	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
5813	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5814	*/
5815	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5816	uint64_t u64GuestCr4 = pCtx->cr4;
5817	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5818	? pCtx->cr4
5819	: CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
5820	Assert(!RT_HI_U32(u64GuestCr4));
5821
5822	/*
5823	* Setup VT-x's view of the guest CR4.
5824	*
5825	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5826	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5827	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5828	*
5829	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5830	*/
5831	if (pVmcsInfo->RealMode.fRealOnV86Active)
5832	{
5833	Assert(pVM->hm.s.vmx.pRealModeTSS);
5834	Assert(PDMVmmDevHeapIsEnabled(pVM));
5835	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5836	}
5837
5838	if (pVM->hm.s.fNestedPaging)
5839	{
5840	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5841	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5842	{
5843	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5844	u64GuestCr4 \|= X86_CR4_PSE;
5845	/* Our identity mapping is a 32-bit page directory. */
5846	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5847	}
5848	/* else use guest CR4.*/
5849	}
5850	else
5851	{
5852	Assert(!pVmxTransient->fIsNestedGuest);
5853
5854	/*
5855	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5856	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5857	*/
5858	switch (pVCpu->hm.s.enmShadowMode)
5859	{
5860	case PGMMODE_REAL: /* Real-mode. */
5861	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5862	case PGMMODE_32_BIT: /* 32-bit paging. */
5863	{
5864	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5865	break;
5866	}
5867
5868	case PGMMODE_PAE: /* PAE paging. */
5869	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5870	{
5871	u64GuestCr4 \|= X86_CR4_PAE;
5872	break;
5873	}
5874
5875	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5876	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5877	{
5878	#ifdef VBOX_WITH_64_BITS_GUESTS
5879	/* For our assumption in hmR0VmxShouldSwapEferMsr. */
5880	Assert(u64GuestCr4 & X86_CR4_PAE);
5881	break;
5882	#endif
5883	}
5884	default:
5885	AssertFailed();
5886	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5887	}
5888	}
5889
5890	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
5891	u64GuestCr4 \|= fSetCr4;
5892	u64GuestCr4 &= fZapCr4;
5893
5894	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5895	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
5896	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
5897
5898	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5899	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5900
5901	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5902
5903	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5904	}
5905	return rc;
5906	}
5907
5908
5909	/**
5910	* Exports the guest debug registers into the guest-state area in the VMCS.
5911	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5912	*
5913	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5914	*
5915	* @returns VBox status code.
5916	* @param pVCpu The cross context virtual CPU structure.
5917	* @param pVmxTransient The VMX-transient structure.
5918	*
5919	* @remarks No-long-jump zone!!!
5920	*/
5921	static int hmR0VmxExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5922	{
5923	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5924
5925	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5926	* stepping. */
5927	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5928	if (pVmxTransient->fIsNestedGuest)
5929	{
5930	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5931	AssertRC(rc);
5932
5933	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5934	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5935	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5936	AssertRC(rc);
5937	return VINF_SUCCESS;
5938	}
5939
5940	#ifdef VBOX_STRICT
5941	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5942	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5943	{
5944	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5945	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5946	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5947	}
5948	#endif
5949
5950	bool fSteppingDB = false;
5951	bool fInterceptMovDRx = false;
5952	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5953	if (pVCpu->hm.s.fSingleInstruction)
5954	{
5955	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5956	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5957	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5958	{
5959	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5960	Assert(fSteppingDB == false);
5961	}
5962	else
5963	{
5964	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5965	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5966	pVCpu->hm.s.fClearTrapFlag = true;
5967	fSteppingDB = true;
5968	}
5969	}
5970
5971	uint64_t u64GuestDr7;
5972	if ( fSteppingDB
5973	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5974	{
5975	/*
5976	* Use the combined guest and host DRx values found in the hypervisor register set
5977	* because the hypervisor debugger has breakpoints active or someone is single stepping
5978	* on the host side without a monitor trap flag.
5979	*
5980	* Note! DBGF expects a clean DR6 state before executing guest code.
5981	*/
5982	if (!CPUMIsHyperDebugStateActive(pVCpu))
5983	{
5984	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5985	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5986	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5987	}
5988
5989	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5990	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
5991	pVCpu->hm.s.fUsingHyperDR7 = true;
5992	fInterceptMovDRx = true;
5993	}
5994	else
5995	{
5996	/*
5997	* If the guest has enabled debug registers, we need to load them prior to
5998	* executing guest code so they'll trigger at the right time.
5999	*/
6000	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
6001	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
6002	{
6003	if (!CPUMIsGuestDebugStateActive(pVCpu))
6004	{
6005	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6006	Assert(CPUMIsGuestDebugStateActive(pVCpu));
6007	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6008	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6009	}
6010	Assert(!fInterceptMovDRx);
6011	}
6012	else if (!CPUMIsGuestDebugStateActive(pVCpu))
6013	{
6014	/*
6015	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
6016	* must intercept #DB in order to maintain a correct DR6 guest value, and
6017	* because we need to intercept it to prevent nested #DBs from hanging the
6018	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
6019	*/
6020	fInterceptMovDRx = true;
6021	}
6022
6023	/* Update DR7 with the actual guest value. */
6024	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
6025	pVCpu->hm.s.fUsingHyperDR7 = false;
6026	}
6027
6028	if (fInterceptMovDRx)
6029	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
6030	else
6031	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
6032
6033	/*
6034	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
6035	* monitor-trap flag and update our cache.
6036	*/
6037	if (uProcCtls != pVmcsInfo->u32ProcCtls)
6038	{
6039	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6040	AssertRC(rc);
6041	pVmcsInfo->u32ProcCtls = uProcCtls;
6042	}
6043
6044	/*
6045	* Update guest DR7.
6046	*/
6047	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
6048	AssertRC(rc);
6049
6050	/*
6051	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6052	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6053	*
6054	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6055	*/
6056	if (fSteppingDB)
6057	{
6058	Assert(pVCpu->hm.s.fSingleInstruction);
6059	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6060
6061	uint32_t fIntrState = 0;
6062	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6063	AssertRC(rc);
6064
6065	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6066	{
6067	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6068	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6069	AssertRC(rc);
6070	}
6071	}
6072
6073	return VINF_SUCCESS;
6074	}
6075
6076
6077	#ifdef VBOX_STRICT
6078	/**
6079	* Strict function to validate segment registers.
6080	*
6081	* @param pVCpu The cross context virtual CPU structure.
6082	* @param pVmcsInfo The VMCS info. object.
6083	*
6084	* @remarks Will import guest CR0 on strict builds during validation of
6085	* segments.
6086	*/
6087	static void hmR0VmxValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
6088	{
6089	/*
6090	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6091	*
6092	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6093	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6094	* unusable bit and doesn't change the guest-context value.
6095	*/
6096	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6097	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6098	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6099	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6100	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6101	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6102	{
6103	/* Protected mode checks */
6104	/* CS */
6105	Assert(pCtx->cs.Attr.n.u1Present);
6106	Assert(!(pCtx->cs.Attr.u & 0xf00));
6107	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6108	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6109	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6110	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6111	\|\| (pCtx->cs.Attr.n.u1Granularity));
6112	/* CS cannot be loaded with NULL in protected mode. */
6113	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6114	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6115	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6116	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6117	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6118	else
6119	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6120	/* SS */
6121	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6122	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6123	if ( !(pCtx->cr0 & X86_CR0_PE)
6124	\|\| pCtx->cs.Attr.n.u4Type == 3)
6125	{
6126	Assert(!pCtx->ss.Attr.n.u2Dpl);
6127	}
6128	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6129	{
6130	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6131	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6132	Assert(pCtx->ss.Attr.n.u1Present);
6133	Assert(!(pCtx->ss.Attr.u & 0xf00));
6134	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6135	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6136	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6137	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6138	\|\| (pCtx->ss.Attr.n.u1Granularity));
6139	}
6140	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6141	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6142	{
6143	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6144	Assert(pCtx->ds.Attr.n.u1Present);
6145	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6146	Assert(!(pCtx->ds.Attr.u & 0xf00));
6147	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6148	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6149	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6150	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6151	\|\| (pCtx->ds.Attr.n.u1Granularity));
6152	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6153	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6154	}
6155	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6156	{
6157	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6158	Assert(pCtx->es.Attr.n.u1Present);
6159	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6160	Assert(!(pCtx->es.Attr.u & 0xf00));
6161	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6162	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6163	\|\| !(pCtx->es.Attr.n.u1Granularity));
6164	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6165	\|\| (pCtx->es.Attr.n.u1Granularity));
6166	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6167	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6168	}
6169	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6170	{
6171	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6172	Assert(pCtx->fs.Attr.n.u1Present);
6173	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6174	Assert(!(pCtx->fs.Attr.u & 0xf00));
6175	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6176	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6177	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6178	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6179	\|\| (pCtx->fs.Attr.n.u1Granularity));
6180	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6181	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6182	}
6183	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6184	{
6185	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6186	Assert(pCtx->gs.Attr.n.u1Present);
6187	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6188	Assert(!(pCtx->gs.Attr.u & 0xf00));
6189	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6190	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6191	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6192	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6193	\|\| (pCtx->gs.Attr.n.u1Granularity));
6194	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6195	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6196	}
6197	/* 64-bit capable CPUs. */
6198	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6199	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6200	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6201	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6202	}
6203	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6204	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6205	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6206	{
6207	/* Real and v86 mode checks. */
6208	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6209	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6210	if (pVmcsInfo->RealMode.fRealOnV86Active)
6211	{
6212	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6213	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6214	}
6215	else
6216	{
6217	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6218	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6219	}
6220
6221	/* CS */
6222	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6223	Assert(pCtx->cs.u32Limit == 0xffff);
6224	Assert(u32CSAttr == 0xf3);
6225	/* SS */
6226	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6227	Assert(pCtx->ss.u32Limit == 0xffff);
6228	Assert(u32SSAttr == 0xf3);
6229	/* DS */
6230	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6231	Assert(pCtx->ds.u32Limit == 0xffff);
6232	Assert(u32DSAttr == 0xf3);
6233	/* ES */
6234	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6235	Assert(pCtx->es.u32Limit == 0xffff);
6236	Assert(u32ESAttr == 0xf3);
6237	/* FS */
6238	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6239	Assert(pCtx->fs.u32Limit == 0xffff);
6240	Assert(u32FSAttr == 0xf3);
6241	/* GS */
6242	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6243	Assert(pCtx->gs.u32Limit == 0xffff);
6244	Assert(u32GSAttr == 0xf3);
6245	/* 64-bit capable CPUs. */
6246	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6247	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6248	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6249	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6250	}
6251	}
6252	#endif /* VBOX_STRICT */
6253
6254
6255	/**
6256	* Exports a guest segment register into the guest-state area in the VMCS.
6257	*
6258	* @returns VBox status code.
6259	* @param pVCpu The cross context virtual CPU structure.
6260	* @param pVmcsInfo The VMCS info. object.
6261	* @param iSegReg The segment register number (X86_SREG_XXX).
6262	* @param pSelReg Pointer to the segment selector.
6263	*
6264	* @remarks No-long-jump zone!!!
6265	*/
6266	static int hmR0VmxExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
6267	{
6268	Assert(iSegReg < X86_SREG_COUNT);
6269
6270	uint32_t u32Access = pSelReg->Attr.u;
6271	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6272	{
6273	/*
6274	* The way to differentiate between whether this is really a null selector or was just
6275	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6276	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6277	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6278	* NULL selectors loaded in protected-mode have their attribute as 0.
6279	*/
6280	if (u32Access)
6281	{ }
6282	else
6283	u32Access = X86DESCATTR_UNUSABLE;
6284	}
6285	else
6286	{
6287	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6288	u32Access = 0xf3;
6289	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6290	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6291	RT_NOREF_PV(pVCpu);
6292	}
6293
6294	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6295	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6296	("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));
6297
6298	/*
6299	* Commit it to the VMCS.
6300	*/
6301	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
6302	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
6303	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
6304	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
6305	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_SEG_SEL(iSegReg), pSelReg->Sel); AssertRC(rc);
6306	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), pSelReg->u32Limit); AssertRC(rc);
6307	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SEG_BASE(iSegReg), pSelReg->u64Base); AssertRC(rc);
6308	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access); AssertRC(rc);
6309	return VINF_SUCCESS;
6310	}
6311
6312
6313	/**
6314	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6315	* area in the VMCS.
6316	*
6317	* @returns VBox status code.
6318	* @param pVCpu The cross context virtual CPU structure.
6319	* @param pVmxTransient The VMX-transient structure.
6320	*
6321	* @remarks Will import guest CR0 on strict builds during validation of
6322	* segments.
6323	* @remarks No-long-jump zone!!!
6324	*/
6325	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6326	{
6327	int rc = VERR_INTERNAL_ERROR_5;
6328	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6329	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6330	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6331
6332	/*
6333	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6334	*/
6335	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6336	{
6337	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6338	{
6339	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6340	if (pVmcsInfo->RealMode.fRealOnV86Active)
6341	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6342	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6343	AssertRC(rc);
6344	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6345	}
6346
6347	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6348	{
6349	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6350	if (pVmcsInfo->RealMode.fRealOnV86Active)
6351	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6352	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6353	AssertRC(rc);
6354	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6355	}
6356
6357	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6358	{
6359	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6360	if (pVmcsInfo->RealMode.fRealOnV86Active)
6361	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6362	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6363	AssertRC(rc);
6364	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6365	}
6366
6367	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6368	{
6369	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6370	if (pVmcsInfo->RealMode.fRealOnV86Active)
6371	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6372	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6373	AssertRC(rc);
6374	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6375	}
6376
6377	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6378	{
6379	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6380	if (pVmcsInfo->RealMode.fRealOnV86Active)
6381	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6382	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6383	AssertRC(rc);
6384	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6385	}
6386
6387	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6388	{
6389	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6390	if (pVmcsInfo->RealMode.fRealOnV86Active)
6391	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6392	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6393	AssertRC(rc);
6394	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6395	}
6396
6397	#ifdef VBOX_STRICT
6398	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6399	#endif
6400	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6401	pCtx->cs.Attr.u));
6402	}
6403
6404	/*
6405	* Guest TR.
6406	*/
6407	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6408	{
6409	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6410
6411	/*
6412	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6413	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6414	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6415	*/
6416	uint16_t u16Sel;
6417	uint32_t u32Limit;
6418	uint64_t u64Base;
6419	uint32_t u32AccessRights;
6420	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6421	{
6422	u16Sel = pCtx->tr.Sel;
6423	u32Limit = pCtx->tr.u32Limit;
6424	u64Base = pCtx->tr.u64Base;
6425	u32AccessRights = pCtx->tr.Attr.u;
6426	}
6427	else
6428	{
6429	Assert(!pVmxTransient->fIsNestedGuest);
6430	Assert(pVM->hm.s.vmx.pRealModeTSS);
6431	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6432
6433	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6434	RTGCPHYS GCPhys;
6435	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6436	AssertRCReturn(rc, rc);
6437
6438	X86DESCATTR DescAttr;
6439	DescAttr.u = 0;
6440	DescAttr.n.u1Present = 1;
6441	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6442
6443	u16Sel = 0;
6444	u32Limit = HM_VTX_TSS_SIZE;
6445	u64Base = GCPhys;
6446	u32AccessRights = DescAttr.u;
6447	}
6448
6449	/* Validate. */
6450	Assert(!(u16Sel & RT_BIT(2)));
6451	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6452	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6453	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6454	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6455	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6456	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6457	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6458	Assert( (u32Limit & 0xfff) == 0xfff
6459	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6460	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6461	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6462
6463	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
6464	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
6465	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
6466	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
6467
6468	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6469	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6470	}
6471
6472	/*
6473	* Guest GDTR.
6474	*/
6475	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6476	{
6477	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6478
6479	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
6480	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
6481
6482	/* Validate. */
6483	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6484
6485	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6486	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6487	}
6488
6489	/*
6490	* Guest LDTR.
6491	*/
6492	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6493	{
6494	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6495
6496	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6497	uint32_t u32Access;
6498	if ( !pVmxTransient->fIsNestedGuest
6499	&& !pCtx->ldtr.Attr.u)
6500	u32Access = X86DESCATTR_UNUSABLE;
6501	else
6502	u32Access = pCtx->ldtr.Attr.u;
6503
6504	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
6505	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
6506	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
6507	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
6508
6509	/* Validate. */
6510	if (!(u32Access & X86DESCATTR_UNUSABLE))
6511	{
6512	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6513	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6514	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6515	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6516	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6517	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6518	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6519	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6520	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6521	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6522	}
6523
6524	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6525	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6526	}
6527
6528	/*
6529	* Guest IDTR.
6530	*/
6531	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6532	{
6533	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6534
6535	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
6536	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
6537
6538	/* Validate. */
6539	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6540
6541	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6542	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6543	}
6544
6545	return VINF_SUCCESS;
6546	}
6547
6548
6549	/**
6550	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6551	* areas.
6552	*
6553	* These MSRs will automatically be loaded to the host CPU on every successful
6554	* VM-entry and stored from the host CPU on every successful VM-exit.
6555	*
6556	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6557	* actual host MSR values are not- updated here for performance reasons. See
6558	* hmR0VmxExportHostMsrs().
6559	*
6560	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6561	*
6562	* @returns VBox status code.
6563	* @param pVCpu The cross context virtual CPU structure.
6564	* @param pVmxTransient The VMX-transient structure.
6565	*
6566	* @remarks No-long-jump zone!!!
6567	*/
6568	static int hmR0VmxExportGuestMsrs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6569	{
6570	AssertPtr(pVCpu);
6571	AssertPtr(pVmxTransient);
6572
6573	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6574	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6575
6576	/*
6577	* MSRs that we use the auto-load/store MSR area in the VMCS.
6578	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
6579	* nothing to do here. The host MSR values are updated when it's safe in
6580	* hmR0VmxLazySaveHostMsrs().
6581	*
6582	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6583	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6584	* emulation. The merged MSR permission bitmap will ensure that we get VM-exits
6585	* for any MSR that are not part of the lazy MSRs so we do not need to place
6586	* those MSRs into the auto-load/store MSR area. Nothing to do here.
6587	*/
6588	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6589	{
6590	/* No auto-load/store MSRs currently. */
6591	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6592	}
6593
6594	/*
6595	* Guest Sysenter MSRs.
6596	*/
6597	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6598	{
6599	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6600
6601	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6602	{
6603	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6604	AssertRC(rc);
6605	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6606	}
6607
6608	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6609	{
6610	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6611	AssertRC(rc);
6612	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6613	}
6614
6615	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6616	{
6617	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6618	AssertRC(rc);
6619	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6620	}
6621	}
6622
6623	/*
6624	* Guest/host EFER MSR.
6625	*/
6626	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6627	{
6628	/* Whether we are using the VMCS to swap the EFER MSR must have been
6629	determined earlier while exporting VM-entry/VM-exit controls. */
6630	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6631	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6632
6633	if (hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
6634	{
6635	/*
6636	* EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
6637	* This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
6638	* VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
6639	* and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
6640	* must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
6641	* the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
6642	* during VM-entry.
6643	*/
6644	uint64_t uGuestEferMsr = pCtx->msrEFER;
6645	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6646	{
6647	if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
6648	uGuestEferMsr &= ~MSR_K6_EFER_LME;
6649	else
6650	Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
6651	}
6652
6653	/*
6654	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6655	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6656	*/
6657	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6658	{
6659	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
6660	AssertRC(rc);
6661	}
6662	else
6663	{
6664	/*
6665	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6666	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6667	*/
6668	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
6669	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6670	AssertRCReturn(rc, rc);
6671	}
6672
6673	Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
6674	}
6675	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6676	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6677
6678	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6679	}
6680
6681	/*
6682	* Other MSRs.
6683	*/
6684	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6685	{
6686	/* Speculation Control (R/W). */
6687	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6688	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6689	{
6690	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6691	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6692	AssertRCReturn(rc, rc);
6693	}
6694
6695	/* Last Branch Record. */
6696	if (pVM->hm.s.vmx.fLbr)
6697	{
6698	uint32_t const idFromIpMsrStart = pVM->hm.s.vmx.idLbrFromIpMsrFirst;
6699	uint32_t const idToIpMsrStart = pVM->hm.s.vmx.idLbrToIpMsrFirst;
6700	uint32_t const cLbrStack = pVM->hm.s.vmx.idLbrFromIpMsrLast - pVM->hm.s.vmx.idLbrFromIpMsrFirst + 1;
6701	Assert(cLbrStack <= 32);
6702	for (uint32_t i = 0; i < cLbrStack; i++)
6703	{
6704	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idFromIpMsrStart + i,
6705	pVmxTransient->pVmcsInfo->au64LbrFromIpMsr[i],
6706	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6707	AssertRCReturn(rc, rc);
6708
6709	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
6710	if (idToIpMsrStart != 0)
6711	{
6712	rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idToIpMsrStart + i,
6713	pVmxTransient->pVmcsInfo->au64LbrToIpMsr[i],
6714	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6715	AssertRCReturn(rc, rc);
6716	}
6717	}
6718
6719	/* Add LBR top-of-stack MSR (which contains the index to the most recent record). */
6720	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, pVM->hm.s.vmx.idLbrTosMsr,
6721	pVmxTransient->pVmcsInfo->u64LbrTosMsr, false /* fSetReadWrite */,
6722	false /* fUpdateHostMsr */);
6723	AssertRCReturn(rc, rc);
6724	}
6725
6726	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6727	}
6728
6729	return VINF_SUCCESS;
6730	}
6731
6732
6733	/**
6734	* Wrapper for running the guest code in VT-x.
6735	*
6736	* @returns VBox status code, no informational status codes.
6737	* @param pVCpu The cross context virtual CPU structure.
6738	* @param pVmxTransient The VMX-transient structure.
6739	*
6740	* @remarks No-long-jump zone!!!
6741	*/
6742	DECLINLINE(int) hmR0VmxRunGuest(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6743	{
6744	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6745	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6746	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6747
6748	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6749
6750	/*
6751	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6752	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6753	* callee-saved and thus the need for this XMM wrapper.
6754	*
6755	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6756	*/
6757	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6758	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6759	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6760	#ifdef VBOX_WITH_KERNEL_USING_XMM
6761	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6762	#else
6763	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu);
6764	#endif
6765	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6766	return rc;
6767	}
6768
6769
6770	/**
6771	* Reports world-switch error and dumps some useful debug info.
6772	*
6773	* @param pVCpu The cross context virtual CPU structure.
6774	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6775	* @param pVmxTransient The VMX-transient structure (only
6776	* exitReason updated).
6777	*/
6778	static void hmR0VmxReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6779	{
6780	Assert(pVCpu);
6781	Assert(pVmxTransient);
6782	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6783
6784	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6785	switch (rcVMRun)
6786	{
6787	case VERR_VMX_INVALID_VMXON_PTR:
6788	AssertFailed();
6789	break;
6790	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6791	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6792	{
6793	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6794	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6795	AssertRC(rc);
6796	hmR0VmxReadExitQualVmcs(pVmxTransient);
6797
6798	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6799	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6800	Cannot do it here as we may have been long preempted. */
6801
6802	#ifdef VBOX_STRICT
6803	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6804	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6805	pVmxTransient->uExitReason));
6806	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6807	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6808	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6809	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6810	else
6811	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6812	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6813	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6814
6815	static struct
6816	{
6817	/** Name of the field to log. */
6818	const char *pszName;
6819	/** The VMCS field. */
6820	uint32_t uVmcsField;
6821	/** Whether host support of this field needs to be checked. */
6822	bool fCheckSupport;
6823	} const s_aVmcsFields[] =
6824	{
6825	{ "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false },
6826	{ "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false },
6827	{ "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true },
6828	{ "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false },
6829	{ "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false },
6830	{ "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false },
6831	{ "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false },
6832	{ "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false },
6833	{ "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false },
6834	{ "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false },
6835	{ "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false },
6836	{ "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false },
6837	{ "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false },
6838	{ "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false },
6839	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false },
6840	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false },
6841	{ "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false },
6842	{ "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false },
6843	{ "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false },
6844	{ "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false },
6845	{ "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true },
6846	{ "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false },
6847	{ "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false },
6848	{ "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false },
6849	{ "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
6850	{ "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false },
6851	{ "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false },
6852	{ "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false },
6853	/* The order of selector fields below are fixed! */
6854	{ "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false },
6855	{ "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false },
6856	{ "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false },
6857	{ "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false },
6858	{ "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false },
6859	{ "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false },
6860	{ "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false },
6861	/* End of ordered selector fields. */
6862	{ "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false },
6863	{ "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false },
6864	{ "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false },
6865	{ "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false },
6866	{ "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false },
6867	{ "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false },
6868	{ "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false },
6869	{ "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false }
6870	};
6871
6872	RTGDTR HostGdtr;
6873	ASMGetGDTR(&HostGdtr);
6874
6875	uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
6876	for (uint32_t i = 0; i < cVmcsFields; i++)
6877	{
6878	uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
6879
6880	bool fSupported;
6881	if (!s_aVmcsFields[i].fCheckSupport)
6882	fSupported = true;
6883	else
6884	{
6885	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6886	switch (uVmcsField)
6887	{
6888	case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hm.s.fNestedPaging; break;
6889	case VMX_VMCS16_VPID: fSupported = pVM->hm.s.vmx.fVpid; break;
6890	case VMX_VMCS32_CTRL_PROC_EXEC2:
6891	fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6892	break;
6893	default:
6894	AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
6895	}
6896	}
6897
6898	if (fSupported)
6899	{
6900	uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
6901	switch (uWidth)
6902	{
6903	case VMX_VMCSFIELD_WIDTH_16BIT:
6904	{
6905	uint16_t u16Val;
6906	rc = VMXReadVmcs16(uVmcsField, &u16Val);
6907	AssertRC(rc);
6908	Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
6909
6910	if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
6911	&& uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
6912	{
6913	if (u16Val < HostGdtr.cbGdt)
6914	{
6915	/* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
6916	static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
6917	"Host FS", "Host GS", "Host TR" };
6918	uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
6919	Assert(idxSel < RT_ELEMENTS(s_apszSel));
6920	PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
6921	hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
6922	}
6923	else
6924	Log4((" Selector value exceeds GDT limit!\n"));
6925	}
6926	break;
6927	}
6928
6929	case VMX_VMCSFIELD_WIDTH_32BIT:
6930	{
6931	uint32_t u32Val;
6932	rc = VMXReadVmcs32(uVmcsField, &u32Val);
6933	AssertRC(rc);
6934	Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
6935	break;
6936	}
6937
6938	case VMX_VMCSFIELD_WIDTH_64BIT:
6939	case VMX_VMCSFIELD_WIDTH_NATURAL:
6940	{
6941	uint64_t u64Val;
6942	rc = VMXReadVmcs64(uVmcsField, &u64Val);
6943	AssertRC(rc);
6944	Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
6945	break;
6946	}
6947	}
6948	}
6949	}
6950
6951	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6952	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6953	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6954	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6955	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6956	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6957	#endif /* VBOX_STRICT */
6958	break;
6959	}
6960
6961	default:
6962	/* Impossible */
6963	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6964	break;
6965	}
6966	}
6967
6968
6969	/**
6970	* Sets up the usage of TSC-offsetting and updates the VMCS.
6971	*
6972	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6973	* VMX-preemption timer.
6974	*
6975	* @returns VBox status code.
6976	* @param pVCpu The cross context virtual CPU structure.
6977	* @param pVmxTransient The VMX-transient structure.
6978	*
6979	* @remarks No-long-jump zone!!!
6980	*/
6981	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6982	{
6983	bool fOffsettedTsc;
6984	bool fParavirtTsc;
6985	uint64_t uTscOffset;
6986	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6987	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6988
6989	if (pVM->hm.s.vmx.fUsePreemptTimer)
6990	{
6991	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6992
6993	/* Make sure the returned values have sane upper and lower boundaries. */
6994	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6995	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6996	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6997	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6998
6999	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
7000	* preemption timers here. We probably need to clamp the preemption timer,
7001	* after converting the timer value to the host. */
7002	uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7003	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7004	AssertRC(rc);
7005	}
7006	else
7007	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7008
7009	if (fParavirtTsc)
7010	{
7011	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7012	information before every VM-entry, hence disable it for performance sake. */
7013	#if 0
7014	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7015	AssertRC(rc);
7016	#endif
7017	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7018	}
7019
7020	if ( fOffsettedTsc
7021	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7022	{
7023	if (pVmxTransient->fIsNestedGuest)
7024	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7025	hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
7026	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7027	}
7028	else
7029	{
7030	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7031	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7032	}
7033	}
7034
7035
7036	/**
7037	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7038	* VM-exit interruption info type.
7039	*
7040	* @returns The IEM exception flags.
7041	* @param uVector The event vector.
7042	* @param uVmxEventType The VMX event type.
7043	*
7044	* @remarks This function currently only constructs flags required for
7045	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7046	* and CR2 aspects of an exception are not included).
7047	*/
7048	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7049	{
7050	uint32_t fIemXcptFlags;
7051	switch (uVmxEventType)
7052	{
7053	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7054	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7055	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7056	break;
7057
7058	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7059	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7060	break;
7061
7062	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7063	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7064	break;
7065
7066	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7067	{
7068	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7069	if (uVector == X86_XCPT_BP)
7070	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7071	else if (uVector == X86_XCPT_OF)
7072	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7073	else
7074	{
7075	fIemXcptFlags = 0;
7076	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7077	}
7078	break;
7079	}
7080
7081	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7082	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7083	break;
7084
7085	default:
7086	fIemXcptFlags = 0;
7087	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7088	break;
7089	}
7090	return fIemXcptFlags;
7091	}
7092
7093
7094	/**
7095	* Sets an event as a pending event to be injected into the guest.
7096	*
7097	* @param pVCpu The cross context virtual CPU structure.
7098	* @param u32IntInfo The VM-entry interruption-information field.
7099	* @param cbInstr The VM-entry instruction length in bytes (for
7100	* software interrupts, exceptions and privileged
7101	* software exceptions).
7102	* @param u32ErrCode The VM-entry exception error code.
7103	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7104	* page-fault.
7105	*/
7106	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7107	RTGCUINTPTR GCPtrFaultAddress)
7108	{
7109	Assert(!pVCpu->hm.s.Event.fPending);
7110	pVCpu->hm.s.Event.fPending = true;
7111	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7112	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7113	pVCpu->hm.s.Event.cbInstr = cbInstr;
7114	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7115	}
7116
7117
7118	/**
7119	* Sets an external interrupt as pending-for-injection into the VM.
7120	*
7121	* @param pVCpu The cross context virtual CPU structure.
7122	* @param u8Interrupt The external interrupt vector.
7123	*/
7124	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
7125	{
7126	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7127	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7128	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7129	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7130	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7131	}
7132
7133
7134	/**
7135	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7136	*
7137	* @param pVCpu The cross context virtual CPU structure.
7138	*/
7139	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPUCC pVCpu)
7140	{
7141	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7142	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7143	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7144	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7145	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7146	}
7147
7148
7149	/**
7150	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7151	*
7152	* @param pVCpu The cross context virtual CPU structure.
7153	*/
7154	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPUCC pVCpu)
7155	{
7156	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7157	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7158	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7159	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7160	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7161	}
7162
7163
7164	/**
7165	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7166	*
7167	* @param pVCpu The cross context virtual CPU structure.
7168	*/
7169	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPUCC pVCpu)
7170	{
7171	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7172	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7173	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7174	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7175	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7176	}
7177
7178
7179	/**
7180	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7181	*
7182	* @param pVCpu The cross context virtual CPU structure.
7183	*/
7184	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPUCC pVCpu)
7185	{
7186	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7187	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7188	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7189	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7190	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7191	}
7192
7193
7194	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7195	/**
7196	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7197	*
7198	* @param pVCpu The cross context virtual CPU structure.
7199	* @param u32ErrCode The error code for the general-protection exception.
7200	*/
7201	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7202	{
7203	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7204	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7205	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7206	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7207	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7208	}
7209
7210
7211	/**
7212	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7213	*
7214	* @param pVCpu The cross context virtual CPU structure.
7215	* @param u32ErrCode The error code for the stack exception.
7216	*/
7217	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7218	{
7219	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7220	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7221	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7222	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7223	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7224	}
7225	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7226
7227
7228	/**
7229	* Fixes up attributes for the specified segment register.
7230	*
7231	* @param pVCpu The cross context virtual CPU structure.
7232	* @param pSelReg The segment register that needs fixing.
7233	* @param pszRegName The register name (for logging and assertions).
7234	*/
7235	static void hmR0VmxFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
7236	{
7237	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7238
7239	/*
7240	* If VT-x marks the segment as unusable, most other bits remain undefined:
7241	* - For CS the L, D and G bits have meaning.
7242	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7243	* - For the remaining data segments no bits are defined.
7244	*
7245	* The present bit and the unusable bit has been observed to be set at the
7246	* same time (the selector was supposed to be invalid as we started executing
7247	* a V8086 interrupt in ring-0).
7248	*
7249	* What should be important for the rest of the VBox code, is that the P bit is
7250	* cleared. Some of the other VBox code recognizes the unusable bit, but
7251	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7252	* safe side here, we'll strip off P and other bits we don't care about. If
7253	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7254	*
7255	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7256	*/
7257	#ifdef VBOX_STRICT
7258	uint32_t const uAttr = pSelReg->Attr.u;
7259	#endif
7260
7261	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7262	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7263	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7264
7265	#ifdef VBOX_STRICT
7266	VMMRZCallRing3Disable(pVCpu);
7267	Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7268	# ifdef DEBUG_bird
7269	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7270	("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7271	pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7272	# endif
7273	VMMRZCallRing3Enable(pVCpu);
7274	NOREF(uAttr);
7275	#endif
7276	RT_NOREF2(pVCpu, pszRegName);
7277	}
7278
7279
7280	/**
7281	* Imports a guest segment register from the current VMCS into the guest-CPU
7282	* context.
7283	*
7284	* @param pVCpu The cross context virtual CPU structure.
7285	* @param iSegReg The segment register number (X86_SREG_XXX).
7286	*
7287	* @remarks Called with interrupts and/or preemption disabled.
7288	*/
7289	static void hmR0VmxImportGuestSegReg(PVMCPUCC pVCpu, uint32_t iSegReg)
7290	{
7291	Assert(iSegReg < X86_SREG_COUNT);
7292	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
7293	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
7294	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
7295	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
7296
7297	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7298
7299	uint16_t u16Sel;
7300	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_SEG_SEL(iSegReg), &u16Sel); AssertRC(rc);
7301	pSelReg->Sel = u16Sel;
7302	pSelReg->ValidSel = u16Sel;
7303
7304	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), &pSelReg->u32Limit); AssertRC(rc);
7305	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SEG_BASE(iSegReg), &pSelReg->u64Base); AssertRC(rc);
7306
7307	uint32_t u32Attr;
7308	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), &u32Attr); AssertRC(rc);
7309	pSelReg->Attr.u = u32Attr;
7310	if (u32Attr & X86DESCATTR_UNUSABLE)
7311	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + iSegReg * 3);
7312
7313	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7314	}
7315
7316
7317	/**
7318	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7319	*
7320	* @param pVCpu The cross context virtual CPU structure.
7321	*
7322	* @remarks Called with interrupts and/or preemption disabled.
7323	*/
7324	static void hmR0VmxImportGuestLdtr(PVMCPUCC pVCpu)
7325	{
7326	uint16_t u16Sel;
7327	uint64_t u64Base;
7328	uint32_t u32Limit, u32Attr;
7329	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
7330	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
7331	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7332	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
7333
7334	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
7335	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
7336	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7337	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7338	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7339	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7340	if (u32Attr & X86DESCATTR_UNUSABLE)
7341	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
7342	}
7343
7344
7345	/**
7346	* Imports the guest TR from the current VMCS into the guest-CPU context.
7347	*
7348	* @param pVCpu The cross context virtual CPU structure.
7349	*
7350	* @remarks Called with interrupts and/or preemption disabled.
7351	*/
7352	static void hmR0VmxImportGuestTr(PVMCPUCC pVCpu)
7353	{
7354	uint16_t u16Sel;
7355	uint64_t u64Base;
7356	uint32_t u32Limit, u32Attr;
7357	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
7358	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
7359	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7360	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
7361
7362	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
7363	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
7364	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7365	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7366	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7367	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7368	/* TR is the only selector that can never be unusable. */
7369	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7370	}
7371
7372
7373	/**
7374	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7375	*
7376	* @param pVCpu The cross context virtual CPU structure.
7377	*
7378	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7379	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7380	* instead!!!
7381	*/
7382	static void hmR0VmxImportGuestRip(PVMCPUCC pVCpu)
7383	{
7384	uint64_t u64Val;
7385	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7386	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7387	{
7388	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
7389	AssertRC(rc);
7390
7391	pCtx->rip = u64Val;
7392	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7393	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7394	}
7395	}
7396
7397
7398	/**
7399	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7400	*
7401	* @param pVCpu The cross context virtual CPU structure.
7402	* @param pVmcsInfo The VMCS info. object.
7403	*
7404	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7405	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7406	* instead!!!
7407	*/
7408	static void hmR0VmxImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7409	{
7410	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7411	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7412	{
7413	uint64_t u64Val;
7414	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7415	AssertRC(rc);
7416
7417	pCtx->rflags.u64 = u64Val;
7418	if (pVmcsInfo->RealMode.fRealOnV86Active)
7419	{
7420	pCtx->eflags.Bits.u1VM = 0;
7421	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7422	}
7423	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7424	}
7425	}
7426
7427
7428	/**
7429	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7430	* context.
7431	*
7432	* @param pVCpu The cross context virtual CPU structure.
7433	* @param pVmcsInfo The VMCS info. object.
7434	*
7435	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7436	* do not log!
7437	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7438	* instead!!!
7439	*/
7440	static void hmR0VmxImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7441	{
7442	uint32_t u32Val;
7443	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
7444	if (!u32Val)
7445	{
7446	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7447	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7448	CPUMSetGuestNmiBlocking(pVCpu, false);
7449	}
7450	else
7451	{
7452	/*
7453	* We must import RIP here to set our EM interrupt-inhibited state.
7454	* We also import RFLAGS as our code that evaluates pending interrupts
7455	* before VM-entry requires it.
7456	*/
7457	hmR0VmxImportGuestRip(pVCpu);
7458	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7459
7460	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7461	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7462	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7463	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7464
7465	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7466	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7467	}
7468	}
7469
7470
7471	/**
7472	* Worker for VMXR0ImportStateOnDemand.
7473	*
7474	* @returns VBox status code.
7475	* @param pVCpu The cross context virtual CPU structure.
7476	* @param pVmcsInfo The VMCS info. object.
7477	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7478	*/
7479	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7480	{
7481	int rc = VINF_SUCCESS;
7482	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7483	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7484	uint32_t u32Val;
7485
7486	/*
7487	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7488	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7489	* neither are other host platforms.
7490	*
7491	* Committing this temporarily as it prevents BSOD.
7492	*
7493	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7494	*/
7495	#ifdef RT_OS_WINDOWS
7496	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7497	return VERR_HM_IPE_1;
7498	#endif
7499
7500	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7501
7502	/*
7503	* We disable interrupts to make the updating of the state and in particular
7504	* the fExtrn modification atomic wrt to preemption hooks.
7505	*/
7506	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7507
7508	fWhat &= pCtx->fExtrn;
7509	if (fWhat)
7510	{
7511	do
7512	{
7513	if (fWhat & CPUMCTX_EXTRN_RIP)
7514	hmR0VmxImportGuestRip(pVCpu);
7515
7516	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7517	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7518
7519	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7520	hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7521
7522	if (fWhat & CPUMCTX_EXTRN_RSP)
7523	{
7524	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &pCtx->rsp);
7525	AssertRC(rc);
7526	}
7527
7528	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7529	{
7530	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7531	if (fWhat & CPUMCTX_EXTRN_CS)
7532	{
7533	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7534	hmR0VmxImportGuestRip(pVCpu);
7535	if (fRealOnV86Active)
7536	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7537	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7538	}
7539	if (fWhat & CPUMCTX_EXTRN_SS)
7540	{
7541	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7542	if (fRealOnV86Active)
7543	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7544	}
7545	if (fWhat & CPUMCTX_EXTRN_DS)
7546	{
7547	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7548	if (fRealOnV86Active)
7549	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7550	}
7551	if (fWhat & CPUMCTX_EXTRN_ES)
7552	{
7553	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7554	if (fRealOnV86Active)
7555	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7556	}
7557	if (fWhat & CPUMCTX_EXTRN_FS)
7558	{
7559	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7560	if (fRealOnV86Active)
7561	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7562	}
7563	if (fWhat & CPUMCTX_EXTRN_GS)
7564	{
7565	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7566	if (fRealOnV86Active)
7567	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7568	}
7569	}
7570
7571	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7572	{
7573	if (fWhat & CPUMCTX_EXTRN_LDTR)
7574	hmR0VmxImportGuestLdtr(pVCpu);
7575
7576	if (fWhat & CPUMCTX_EXTRN_GDTR)
7577	{
7578	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
7579	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
7580	pCtx->gdtr.cbGdt = u32Val;
7581	}
7582
7583	/* Guest IDTR. */
7584	if (fWhat & CPUMCTX_EXTRN_IDTR)
7585	{
7586	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
7587	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
7588	pCtx->idtr.cbIdt = u32Val;
7589	}
7590
7591	/* Guest TR. */
7592	if (fWhat & CPUMCTX_EXTRN_TR)
7593	{
7594	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7595	don't need to import that one. */
7596	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7597	hmR0VmxImportGuestTr(pVCpu);
7598	}
7599	}
7600
7601	if (fWhat & CPUMCTX_EXTRN_DR7)
7602	{
7603	if (!pVCpu->hm.s.fUsingHyperDR7)
7604	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &pCtx->dr[7]); AssertRC(rc);
7605	}
7606
7607	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7608	{
7609	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
7610	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
7611	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
7612	pCtx->SysEnter.cs = u32Val;
7613	}
7614
7615	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7616	{
7617	if ( pVM->hm.s.fAllow64BitGuests
7618	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7619	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7620	}
7621
7622	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7623	{
7624	if ( pVM->hm.s.fAllow64BitGuests
7625	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7626	{
7627	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7628	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7629	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7630	}
7631	}
7632
7633	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7634	{
7635	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7636	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7637	Assert(pMsrs);
7638	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7639	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7640	for (uint32_t i = 0; i < cMsrs; i++)
7641	{
7642	uint32_t const idMsr = pMsrs[i].u32Msr;
7643	switch (idMsr)
7644	{
7645	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7646	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7647	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7648	default:
7649	{
7650	uint32_t idxLbrMsr;
7651	if (pVM->hm.s.vmx.fLbr)
7652	{
7653	if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
7654	{
7655	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfo->au64LbrFromIpMsr));
7656	pVmcsInfo->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
7657	break;
7658	}
7659	else if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
7660	{
7661	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfo->au64LbrFromIpMsr));
7662	pVmcsInfo->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
7663	break;
7664	}
7665	else if (idMsr == pVM->hm.s.vmx.idLbrTosMsr)
7666	{
7667	pVmcsInfo->u64LbrTosMsr = pMsrs[i].u64Value;
7668	break;
7669	}
7670	/* Fallthru (no break) */
7671	}
7672	pCtx->fExtrn = 0;
7673	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7674	ASMSetFlags(fEFlags);
7675	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7676	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7677	}
7678	}
7679	}
7680	}
7681
7682	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7683	{
7684	if (fWhat & CPUMCTX_EXTRN_CR0)
7685	{
7686	uint64_t u64Cr0;
7687	uint64_t u64Shadow;
7688	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
7689	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
7690	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7691	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7692	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7693	#else
7694	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7695	{
7696	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7697	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7698	}
7699	else
7700	{
7701	/*
7702	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
7703	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7704	* re-construct CR0. See @bugref{9180#c95} for details.
7705	*/
7706	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7707	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7708	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7709	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
7710	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
7711	}
7712	#endif
7713	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7714	CPUMSetGuestCR0(pVCpu, u64Cr0);
7715	VMMRZCallRing3Enable(pVCpu);
7716	}
7717
7718	if (fWhat & CPUMCTX_EXTRN_CR4)
7719	{
7720	uint64_t u64Cr4;
7721	uint64_t u64Shadow;
7722	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
7723	rc \|= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
7724	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7725	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7726	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7727	#else
7728	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7729	{
7730	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7731	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7732	}
7733	else
7734	{
7735	/*
7736	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
7737	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7738	* re-construct CR4. See @bugref{9180#c95} for details.
7739	*/
7740	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7741	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7742	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7743	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
7744	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
7745	}
7746	#endif
7747	pCtx->cr4 = u64Cr4;
7748	}
7749
7750	if (fWhat & CPUMCTX_EXTRN_CR3)
7751	{
7752	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7753	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7754	\|\| ( pVM->hm.s.fNestedPaging
7755	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7756	{
7757	uint64_t u64Cr3;
7758	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
7759	if (pCtx->cr3 != u64Cr3)
7760	{
7761	pCtx->cr3 = u64Cr3;
7762	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7763	}
7764
7765	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7766	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7767	if (CPUMIsGuestInPAEModeEx(pCtx))
7768	{
7769	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
7770	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
7771	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
7772	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
7773	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7774	}
7775	}
7776	}
7777	}
7778
7779	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7780	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7781	{
7782	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7783	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7784	{
7785	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7786	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7787	if (RT_SUCCESS(rc))
7788	{ /* likely */ }
7789	else
7790	break;
7791	}
7792	}
7793	#endif
7794	} while (0);
7795
7796	if (RT_SUCCESS(rc))
7797	{
7798	/* Update fExtrn. */
7799	pCtx->fExtrn &= ~fWhat;
7800
7801	/* If everything has been imported, clear the HM keeper bit. */
7802	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7803	{
7804	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7805	Assert(!pCtx->fExtrn);
7806	}
7807	}
7808	}
7809	else
7810	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7811
7812	/*
7813	* Restore interrupts.
7814	*/
7815	ASMSetFlags(fEFlags);
7816
7817	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7818
7819	if (RT_SUCCESS(rc))
7820	{ /* likely */ }
7821	else
7822	return rc;
7823
7824	/*
7825	* Honor any pending CR3 updates.
7826	*
7827	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
7828	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7829	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7830	*
7831	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7832	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7833	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7834	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7835	*
7836	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7837	*/
7838	if (VMMRZCallRing3IsEnabled(pVCpu))
7839	{
7840	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7841	{
7842	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7843	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7844	}
7845
7846	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7847	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7848
7849	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7850	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7851	}
7852
7853	return VINF_SUCCESS;
7854	}
7855
7856
7857	/**
7858	* Saves the guest state from the VMCS into the guest-CPU context.
7859	*
7860	* @returns VBox status code.
7861	* @param pVCpu The cross context virtual CPU structure.
7862	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7863	*/
7864	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
7865	{
7866	AssertPtr(pVCpu);
7867	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7868	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7869	}
7870
7871
7872	/**
7873	* Check per-VM and per-VCPU force flag actions that require us to go back to
7874	* ring-3 for one reason or another.
7875	*
7876	* @returns Strict VBox status code (i.e. informational status codes too)
7877	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7878	* ring-3.
7879	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7880	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7881	* interrupts)
7882	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7883	* all EMTs to be in ring-3.
7884	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7885	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7886	* to the EM loop.
7887	*
7888	* @param pVCpu The cross context virtual CPU structure.
7889	* @param pVmxTransient The VMX-transient structure.
7890	* @param fStepping Whether we are single-stepping the guest using the
7891	* hypervisor debugger.
7892	*
7893	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7894	* is no longer in VMX non-root mode.
7895	*/
7896	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, bool fStepping)
7897	{
7898	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7899
7900	/*
7901	* Update pending interrupts into the APIC's IRR.
7902	*/
7903	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7904	APICUpdatePendingInterrupts(pVCpu);
7905
7906	/*
7907	* Anything pending? Should be more likely than not if we're doing a good job.
7908	*/
7909	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7910	if ( !fStepping
7911	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7912	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7913	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7914	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7915	return VINF_SUCCESS;
7916
7917	/* Pending PGM C3 sync. */
7918	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7919	{
7920	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7921	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7922	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7923	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7924	if (rcStrict != VINF_SUCCESS)
7925	{
7926	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7927	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7928	return rcStrict;
7929	}
7930	}
7931
7932	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7933	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7934	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7935	{
7936	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7937	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7938	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7939	return rc;
7940	}
7941
7942	/* Pending VM request packets, such as hardware interrupts. */
7943	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7944	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7945	{
7946	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7947	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7948	return VINF_EM_PENDING_REQUEST;
7949	}
7950
7951	/* Pending PGM pool flushes. */
7952	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7953	{
7954	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7955	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7956	return VINF_PGM_POOL_FLUSH_PENDING;
7957	}
7958
7959	/* Pending DMA requests. */
7960	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7961	{
7962	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7963	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7964	return VINF_EM_RAW_TO_R3;
7965	}
7966
7967	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7968	/*
7969	* Pending nested-guest events.
7970	*
7971	* Please note the priority of these events are specified and important.
7972	* See Intel spec. 29.4.3.2 "APIC-Write Emulation".
7973	* See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
7974	*/
7975	if (pVmxTransient->fIsNestedGuest)
7976	{
7977	/* Pending nested-guest APIC-write. */
7978	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7979	{
7980	Log4Func(("Pending nested-guest APIC-write\n"));
7981	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
7982	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7983	return rcStrict;
7984	}
7985
7986	/* Pending nested-guest monitor-trap flag (MTF). */
7987	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
7988	{
7989	Log4Func(("Pending nested-guest MTF\n"));
7990	VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
7991	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7992	return rcStrict;
7993	}
7994
7995	/* Pending nested-guest VMX-preemption timer expired. */
7996	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
7997	{
7998	Log4Func(("Pending nested-guest MTF\n"));
7999	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
8000	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8001	return rcStrict;
8002	}
8003	}
8004	#else
8005	NOREF(pVmxTransient);
8006	#endif
8007
8008	return VINF_SUCCESS;
8009	}
8010
8011
8012	/**
8013	* Converts any TRPM trap into a pending HM event. This is typically used when
8014	* entering from ring-3 (not longjmp returns).
8015	*
8016	* @param pVCpu The cross context virtual CPU structure.
8017	*/
8018	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
8019	{
8020	Assert(TRPMHasTrap(pVCpu));
8021	Assert(!pVCpu->hm.s.Event.fPending);
8022
8023	uint8_t uVector;
8024	TRPMEVENT enmTrpmEvent;
8025	uint32_t uErrCode;
8026	RTGCUINTPTR GCPtrFaultAddress;
8027	uint8_t cbInstr;
8028	bool fIcebp;
8029
8030	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
8031	AssertRC(rc);
8032
8033	uint32_t u32IntInfo;
8034	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
8035	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
8036
8037	rc = TRPMResetTrap(pVCpu);
8038	AssertRC(rc);
8039	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8040	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8041
8042	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8043	}
8044
8045
8046	/**
8047	* Converts the pending HM event into a TRPM trap.
8048	*
8049	* @param pVCpu The cross context virtual CPU structure.
8050	*/
8051	static void hmR0VmxPendingEventToTrpmTrap(PVMCPUCC pVCpu)
8052	{
8053	Assert(pVCpu->hm.s.Event.fPending);
8054
8055	/* If a trap was already pending, we did something wrong! */
8056	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8057
8058	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
8059	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
8060	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
8061
8062	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8063
8064	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8065	AssertRC(rc);
8066
8067	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8068	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
8069
8070	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
8071	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8072	else
8073	{
8074	uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
8075	switch (uVectorType)
8076	{
8077	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
8078	TRPMSetTrapDueToIcebp(pVCpu);
8079	RT_FALL_THRU();
8080	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8081	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
8082	{
8083	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8084	\|\| ( uVector == X86_XCPT_BP /* INT3 */
8085	\|\| uVector == X86_XCPT_OF /* INTO */
8086	\|\| uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
8087	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
8088	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8089	break;
8090	}
8091	}
8092	}
8093
8094	/* We're now done converting the pending event. */
8095	pVCpu->hm.s.Event.fPending = false;
8096	}
8097
8098
8099	/**
8100	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8101	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8102	*
8103	* @param pVCpu The cross context virtual CPU structure.
8104	* @param pVmcsInfo The VMCS info. object.
8105	*/
8106	static void hmR0VmxSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
8107	{
8108	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8109	{
8110	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8111	{
8112	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8113	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8114	AssertRC(rc);
8115	}
8116	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8117	}
8118
8119
8120	/**
8121	* Clears the interrupt-window exiting control in the VMCS.
8122	*
8123	* @param pVmcsInfo The VMCS info. object.
8124	*/
8125	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8126	{
8127	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8128	{
8129	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8130	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8131	AssertRC(rc);
8132	}
8133	}
8134
8135
8136	/**
8137	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8138	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8139	*
8140	* @param pVCpu The cross context virtual CPU structure.
8141	* @param pVmcsInfo The VMCS info. object.
8142	*/
8143	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
8144	{
8145	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8146	{
8147	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8148	{
8149	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8150	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8151	AssertRC(rc);
8152	Log4Func(("Setup NMI-window exiting\n"));
8153	}
8154	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8155	}
8156
8157
8158	/**
8159	* Clears the NMI-window exiting control in the VMCS.
8160	*
8161	* @param pVmcsInfo The VMCS info. object.
8162	*/
8163	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8164	{
8165	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8166	{
8167	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8168	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8169	AssertRC(rc);
8170	}
8171	}
8172
8173
8174	/**
8175	* Does the necessary state syncing before returning to ring-3 for any reason
8176	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8177	*
8178	* @returns VBox status code.
8179	* @param pVCpu The cross context virtual CPU structure.
8180	* @param fImportState Whether to import the guest state from the VMCS back
8181	* to the guest-CPU context.
8182	*
8183	* @remarks No-long-jmp zone!!!
8184	*/
8185	static int hmR0VmxLeave(PVMCPUCC pVCpu, bool fImportState)
8186	{
8187	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8188	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8189
8190	RTCPUID const idCpu = RTMpCpuId();
8191	Log4Func(("HostCpuId=%u\n", idCpu));
8192
8193	/*
8194	* !!! IMPORTANT !!!
8195	* If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
8196	*/
8197
8198	/* Save the guest state if necessary. */
8199	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8200	if (fImportState)
8201	{
8202	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8203	AssertRCReturn(rc, rc);
8204	}
8205
8206	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8207	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8208	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8209
8210	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8211	#ifdef VBOX_STRICT
8212	if (CPUMIsHyperDebugStateActive(pVCpu))
8213	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8214	#endif
8215	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8216	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
8217	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
8218
8219	/* Restore host-state bits that VT-x only restores partially. */
8220	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8221	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8222	{
8223	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8224	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8225	}
8226	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8227
8228	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8229	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8230	{
8231	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8232	if (!fImportState)
8233	{
8234	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8235	AssertRCReturn(rc, rc);
8236	}
8237	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8238	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8239	}
8240	else
8241	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8242
8243	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8244	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8245
8246	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8247	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8248	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8249	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8250	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8251	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8252	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8253	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8254	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8255	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8256
8257	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8258
8259	/** @todo This partially defeats the purpose of having preemption hooks.
8260	* The problem is, deregistering the hooks should be moved to a place that
8261	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8262	* context.
8263	*/
8264	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8265	AssertRCReturn(rc, rc);
8266
8267	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8268	/*
8269	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8270	* clear a shadow VMCS before allowing that VMCS to become active on another
8271	* logical processor. We may or may not be importing guest state which clears
8272	* it, so cover for it here.
8273	*
8274	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8275	*/
8276	if ( pVmcsInfo->pvShadowVmcs
8277	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8278	{
8279	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8280	AssertRCReturn(rc, rc);
8281	}
8282
8283	/*
8284	* Flag that we need to re-export the host state if we switch to this VMCS before
8285	* executing guest or nested-guest code.
8286	*/
8287	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
8288	#endif
8289
8290	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8291	NOREF(idCpu);
8292	return VINF_SUCCESS;
8293	}
8294
8295
8296	/**
8297	* Leaves the VT-x session.
8298	*
8299	* @returns VBox status code.
8300	* @param pVCpu The cross context virtual CPU structure.
8301	*
8302	* @remarks No-long-jmp zone!!!
8303	*/
8304	static int hmR0VmxLeaveSession(PVMCPUCC pVCpu)
8305	{
8306	HM_DISABLE_PREEMPT(pVCpu);
8307	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8308	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8309	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8310
8311	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8312	and done this from the VMXR0ThreadCtxCallback(). */
8313	if (!pVCpu->hm.s.fLeaveDone)
8314	{
8315	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8316	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8317	pVCpu->hm.s.fLeaveDone = true;
8318	}
8319	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8320
8321	/*
8322	* !!! IMPORTANT !!!
8323	* If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
8324	*/
8325
8326	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8327	/** @todo Deregistering here means we need to VMCLEAR always
8328	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8329	* for calling VMMR0ThreadCtxHookDisable here! */
8330	VMMR0ThreadCtxHookDisable(pVCpu);
8331
8332	/* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
8333	int rc = HMR0LeaveCpu(pVCpu);
8334	HM_RESTORE_PREEMPT();
8335	return rc;
8336	}
8337
8338
8339	/**
8340	* Does the necessary state syncing before doing a longjmp to ring-3.
8341	*
8342	* @returns VBox status code.
8343	* @param pVCpu The cross context virtual CPU structure.
8344	*
8345	* @remarks No-long-jmp zone!!!
8346	*/
8347	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPUCC pVCpu)
8348	{
8349	return hmR0VmxLeaveSession(pVCpu);
8350	}
8351
8352
8353	/**
8354	* Take necessary actions before going back to ring-3.
8355	*
8356	* An action requires us to go back to ring-3. This function does the necessary
8357	* steps before we can safely return to ring-3. This is not the same as longjmps
8358	* to ring-3, this is voluntary and prepares the guest so it may continue
8359	* executing outside HM (recompiler/IEM).
8360	*
8361	* @returns VBox status code.
8362	* @param pVCpu The cross context virtual CPU structure.
8363	* @param rcExit The reason for exiting to ring-3. Can be
8364	* VINF_VMM_UNKNOWN_RING3_CALL.
8365	*/
8366	static int hmR0VmxExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
8367	{
8368	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8369
8370	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8371	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8372	{
8373	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8374	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8375	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8376	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8377	}
8378
8379	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8380	VMMRZCallRing3Disable(pVCpu);
8381	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8382
8383	/*
8384	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8385	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8386	*
8387	* This is because execution may continue from ring-3 and we would need to inject
8388	* the event from there (hence place it back in TRPM).
8389	*/
8390	if (pVCpu->hm.s.Event.fPending)
8391	{
8392	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8393	Assert(!pVCpu->hm.s.Event.fPending);
8394
8395	/* Clear the events from the VMCS. */
8396	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
8397	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
8398	}
8399	#ifdef VBOX_STRICT
8400	/*
8401	* We check for rcExit here since for errors like VERR_VMX_UNABLE_TO_START_VM (which are
8402	* fatal), we don't care about verifying duplicate injection of events. Errors like
8403	* VERR_EM_INTERPRET are converted to their VINF_* counterparts -prior- to calling this
8404	* function so those should and will be checked below.
8405	*/
8406	else if (RT_SUCCESS(rcExit))
8407	{
8408	/*
8409	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8410	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8411	* occasionally, see @bugref{9180#c42}.
8412	*
8413	* However, if the VM-entry failed, any VM entry-interruption info. field would
8414	* be left unmodified as the event would not have been injected to the guest. In
8415	* such cases, don't assert, we're not going to continue guest execution anyway.
8416	*/
8417	uint32_t uExitReason;
8418	uint32_t uEntryIntInfo;
8419	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8420	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8421	AssertRC(rc);
8422	AssertMsg(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo),
8423	("uExitReason=%#RX32 uEntryIntInfo=%#RX32 rcExit=%d\n", uExitReason, uEntryIntInfo, VBOXSTRICTRC_VAL(rcExit)));
8424	}
8425	#endif
8426
8427	/*
8428	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8429	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8430	* (e.g. TPR below threshold).
8431	*/
8432	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8433	{
8434	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8435	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8436	}
8437
8438	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8439	and if we're injecting an event we should have a TRPM trap pending. */
8440	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8441	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8442	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8443	#endif
8444
8445	/* Save guest state and restore host state bits. */
8446	int rc = hmR0VmxLeaveSession(pVCpu);
8447	AssertRCReturn(rc, rc);
8448	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8449
8450	/* Thread-context hooks are unregistered at this point!!! */
8451	/* Ring-3 callback notifications are unregistered at this point!!! */
8452
8453	/* Sync recompiler state. */
8454	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8455	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8456	\| CPUM_CHANGED_LDTR
8457	\| CPUM_CHANGED_GDTR
8458	\| CPUM_CHANGED_IDTR
8459	\| CPUM_CHANGED_TR
8460	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8461	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8462	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8463	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8464
8465	Assert(!pVCpu->hm.s.fClearTrapFlag);
8466
8467	/* Update the exit-to-ring 3 reason. */
8468	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8469
8470	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8471	if ( rcExit != VINF_EM_RAW_INTERRUPT
8472	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8473	{
8474	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8475	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8476	}
8477
8478	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8479	VMMRZCallRing3Enable(pVCpu);
8480	return rc;
8481	}
8482
8483
8484	/**
8485	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8486	* longjump to ring-3 and possibly get preempted.
8487	*
8488	* @returns VBox status code.
8489	* @param pVCpu The cross context virtual CPU structure.
8490	* @param enmOperation The operation causing the ring-3 longjump.
8491	*/
8492	VMMR0DECL(int) VMXR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
8493	{
8494	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8495	{
8496	/*
8497	* !!! IMPORTANT !!!
8498	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8499	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8500	*/
8501	VMMRZCallRing3RemoveNotification(pVCpu);
8502	VMMRZCallRing3Disable(pVCpu);
8503	HM_DISABLE_PREEMPT(pVCpu);
8504
8505	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8506	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8507	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8508	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8509
8510	/* Restore host-state bits that VT-x only restores partially. */
8511	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8512	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8513	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8514	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8515
8516	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8517	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8518	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8519
8520	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8521	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8522	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8523
8524	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8525	cleared as part of importing the guest state above. */
8526	hmR0VmxClearVmcs(pVmcsInfo);
8527
8528	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8529	VMMR0ThreadCtxHookDisable(pVCpu);
8530
8531	/* Leave HM context. This takes care of local init (term). */
8532	HMR0LeaveCpu(pVCpu);
8533	HM_RESTORE_PREEMPT();
8534	return VINF_SUCCESS;
8535	}
8536
8537	Assert(pVCpu);
8538	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8539	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8540
8541	VMMRZCallRing3Disable(pVCpu);
8542	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8543
8544	Log4Func(("-> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8545
8546	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8547	AssertRCReturn(rc, rc);
8548
8549	VMMRZCallRing3Enable(pVCpu);
8550	return VINF_SUCCESS;
8551	}
8552
8553
8554	/**
8555	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8556	* stack.
8557	*
8558	* @returns Strict VBox status code (i.e. informational status codes too).
8559	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8560	* @param pVCpu The cross context virtual CPU structure.
8561	* @param uValue The value to push to the guest stack.
8562	*/
8563	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
8564	{
8565	/*
8566	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8567	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8568	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8569	*/
8570	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8571	if (pCtx->sp == 1)
8572	return VINF_EM_RESET;
8573	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8574	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8575	AssertRC(rc);
8576	return rc;
8577	}
8578
8579
8580	/**
8581	* Injects an event into the guest upon VM-entry by updating the relevant fields
8582	* in the VM-entry area in the VMCS.
8583	*
8584	* @returns Strict VBox status code (i.e. informational status codes too).
8585	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8586	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8587	*
8588	* @param pVCpu The cross context virtual CPU structure.
8589	* @param pVmxTransient The VMX-transient structure.
8590	* @param pEvent The event being injected.
8591	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8592	* will be updated if necessary. This cannot not be NULL.
8593	* @param fStepping Whether we're single-stepping guest execution and should
8594	* return VINF_EM_DBG_STEPPED if the event is injected
8595	* directly (registers modified by us, not by hardware on
8596	* VM-entry).
8597	*/
8598	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8599	uint32_t *pfIntrState)
8600	{
8601	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8602	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8603	Assert(pfIntrState);
8604
8605	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8606	uint32_t u32IntInfo = pEvent->u64IntInfo;
8607	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8608	uint32_t const cbInstr = pEvent->cbInstr;
8609	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8610	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8611	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8612
8613	#ifdef VBOX_STRICT
8614	/*
8615	* Validate the error-code-valid bit for hardware exceptions.
8616	* No error codes for exceptions in real-mode.
8617	*
8618	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8619	*/
8620	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8621	&& !CPUMIsGuestInRealModeEx(pCtx))
8622	{
8623	switch (uVector)
8624	{
8625	case X86_XCPT_PF:
8626	case X86_XCPT_DF:
8627	case X86_XCPT_TS:
8628	case X86_XCPT_NP:
8629	case X86_XCPT_SS:
8630	case X86_XCPT_GP:
8631	case X86_XCPT_AC:
8632	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8633	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8634	RT_FALL_THRU();
8635	default:
8636	break;
8637	}
8638	}
8639
8640	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8641	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8642	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8643	#endif
8644
8645	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8646	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
8647	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
8648	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
8649	{
8650	Assert(uVector <= X86_XCPT_LAST);
8651	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI \|\| uVector == X86_XCPT_NMI);
8652	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT \|\| uVector == X86_XCPT_DB);
8653	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedXcptsR0[uVector]);
8654	}
8655	else
8656	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8657
8658	/*
8659	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8660	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8661	* interrupt handler in the (real-mode) guest.
8662	*
8663	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8664	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8665	*/
8666	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8667	{
8668	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8669	{
8670	/*
8671	* For CPUs with unrestricted guest execution enabled and with the guest
8672	* in real-mode, we must not set the deliver-error-code bit.
8673	*
8674	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8675	*/
8676	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8677	}
8678	else
8679	{
8680	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8681	Assert(PDMVmmDevHeapIsEnabled(pVM));
8682	Assert(pVM->hm.s.vmx.pRealModeTSS);
8683	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8684
8685	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8686	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8687	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8688	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8689	AssertRCReturn(rc2, rc2);
8690
8691	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8692	size_t const cbIdtEntry = sizeof(X86IDTR16);
8693	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8694	{
8695	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8696	if (uVector == X86_XCPT_DF)
8697	return VINF_EM_RESET;
8698
8699	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8700	No error codes for exceptions in real-mode. */
8701	if (uVector == X86_XCPT_GP)
8702	{
8703	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8704	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8705	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8706	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8707	HMEVENT EventXcptDf;
8708	RT_ZERO(EventXcptDf);
8709	EventXcptDf.u64IntInfo = uXcptDfInfo;
8710	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8711	}
8712
8713	/*
8714	* If we're injecting an event with no valid IDT entry, inject a #GP.
8715	* No error codes for exceptions in real-mode.
8716	*
8717	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8718	*/
8719	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8720	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8721	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8722	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8723	HMEVENT EventXcptGp;
8724	RT_ZERO(EventXcptGp);
8725	EventXcptGp.u64IntInfo = uXcptGpInfo;
8726	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8727	}
8728
8729	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8730	uint16_t uGuestIp = pCtx->ip;
8731	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8732	{
8733	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8734	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8735	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8736	}
8737	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8738	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8739
8740	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8741	X86IDTR16 IdtEntry;
8742	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8743	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8744	AssertRCReturn(rc2, rc2);
8745
8746	/* Construct the stack frame for the interrupt/exception handler. */
8747	VBOXSTRICTRC rcStrict;
8748	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8749	if (rcStrict == VINF_SUCCESS)
8750	{
8751	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8752	if (rcStrict == VINF_SUCCESS)
8753	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8754	}
8755
8756	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8757	if (rcStrict == VINF_SUCCESS)
8758	{
8759	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8760	pCtx->rip = IdtEntry.offSel;
8761	pCtx->cs.Sel = IdtEntry.uSel;
8762	pCtx->cs.ValidSel = IdtEntry.uSel;
8763	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8764	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8765	&& uVector == X86_XCPT_PF)
8766	pCtx->cr2 = GCPtrFault;
8767
8768	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8769	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8770	\| HM_CHANGED_GUEST_RSP);
8771
8772	/*
8773	* If we delivered a hardware exception (other than an NMI) and if there was
8774	* block-by-STI in effect, we should clear it.
8775	*/
8776	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8777	{
8778	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8779	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8780	Log4Func(("Clearing inhibition due to STI\n"));
8781	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8782	}
8783
8784	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8785	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8786
8787	/*
8788	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8789	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8790	*/
8791	pVCpu->hm.s.Event.fPending = false;
8792
8793	/*
8794	* If we eventually support nested-guest execution without unrestricted guest execution,
8795	* we should set fInterceptEvents here.
8796	*/
8797	Assert(!pVmxTransient->fIsNestedGuest);
8798
8799	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8800	if (fStepping)
8801	rcStrict = VINF_EM_DBG_STEPPED;
8802	}
8803	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8804	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8805	return rcStrict;
8806	}
8807	}
8808
8809	/*
8810	* Validate.
8811	*/
8812	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8813	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8814
8815	/*
8816	* Inject the event into the VMCS.
8817	*/
8818	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8819	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8820	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8821	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8822	AssertRC(rc);
8823
8824	/*
8825	* Update guest CR2 if this is a page-fault.
8826	*/
8827	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8828	pCtx->cr2 = GCPtrFault;
8829
8830	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8831	return VINF_SUCCESS;
8832	}
8833
8834
8835	/**
8836	* Evaluates the event to be delivered to the guest and sets it as the pending
8837	* event.
8838	*
8839	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
8840	* exits to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must
8841	* NOT restore these force-flags.
8842	*
8843	* @returns Strict VBox status code (i.e. informational status codes too).
8844	* @param pVCpu The cross context virtual CPU structure.
8845	* @param pVmxTransient The VMX-transient structure.
8846	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8847	*/
8848	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8849	{
8850	Assert(pfIntrState);
8851	Assert(!TRPMHasTrap(pVCpu));
8852
8853	/*
8854	* Compute/update guest-interruptibility state related FFs.
8855	* The FFs will be used below while evaluating events to be injected.
8856	*/
8857	*pfIntrState = hmR0VmxGetGuestIntrStateAndUpdateFFs(pVCpu);
8858
8859	/*
8860	* Evaluate if a new event needs to be injected.
8861	* An event that's already pending has already performed all necessary checks.
8862	*/
8863	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8864	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8865	if ( !pVCpu->hm.s.Event.fPending
8866	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
8867	{
8868	/** @todo SMI. SMIs take priority over NMIs. */
8869
8870	/*
8871	* NMIs.
8872	* NMIs take priority over external interrupts.
8873	*/
8874	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8875	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8876	{
8877	/*
8878	* For a guest, the FF always indicates the guest's ability to receive an NMI.
8879	*
8880	* For a nested-guest, the FF always indicates the outer guest's ability to
8881	* receive an NMI while the guest-interruptibility state bit depends on whether
8882	* the nested-hypervisor is using virtual-NMIs.
8883	*/
8884	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
8885	{
8886	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8887	if ( fIsNestedGuest
8888	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
8889	return IEMExecVmxVmexitXcptNmi(pVCpu);
8890	#endif
8891	hmR0VmxSetPendingXcptNmi(pVCpu);
8892	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8893	Log4Func(("NMI pending injection\n"));
8894
8895	/* We've injected the NMI, bail. */
8896	return VINF_SUCCESS;
8897	}
8898	else if (!fIsNestedGuest)
8899	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8900	}
8901
8902	/*
8903	* External interrupts (PIC/APIC).
8904	* Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it.
8905	* We cannot re-request the interrupt from the controller again.
8906	*/
8907	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8908	&& !pVCpu->hm.s.fSingleInstruction)
8909	{
8910	Assert(!DBGFIsStepping(pVCpu));
8911	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8912	AssertRC(rc);
8913
8914	/*
8915	* We must not check EFLAGS directly when executing a nested-guest, use
8916	* CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of
8917	* external interrupts when "External interrupt exiting" is set. This fixes a nasty
8918	* SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by
8919	* other VM-exits (like a preemption timer), see @bugref{9562#c18}.
8920	*
8921	* See Intel spec. 25.4.1 "Event Blocking".
8922	*/
8923	if (CPUMIsGuestPhysIntrEnabled(pVCpu))
8924	{
8925	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8926	if ( fIsNestedGuest
8927	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8928	{
8929	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8930	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8931	return rcStrict;
8932	}
8933	#endif
8934	uint8_t u8Interrupt;
8935	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8936	if (RT_SUCCESS(rc))
8937	{
8938	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8939	if ( fIsNestedGuest
8940	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8941	{
8942	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8943	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8944	return rcStrict;
8945	}
8946	#endif
8947	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8948	Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
8949	}
8950	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8951	{
8952	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8953
8954	if ( !fIsNestedGuest
8955	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8956	hmR0VmxApicSetTprThreshold(pVmcsInfo, u8Interrupt >> 4);
8957	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8958
8959	/*
8960	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8961	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8962	* need to re-set this force-flag here.
8963	*/
8964	}
8965	else
8966	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8967
8968	/* We've injected the interrupt or taken necessary action, bail. */
8969	return VINF_SUCCESS;
8970	}
8971	else if (!fIsNestedGuest)
8972	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8973	}
8974	}
8975	else if (!fIsNestedGuest)
8976	{
8977	/*
8978	* An event is being injected or we are in an interrupt shadow. Check if another event is
8979	* pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept
8980	* the pending event.
8981	*/
8982	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8983	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8984	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8985	&& !pVCpu->hm.s.fSingleInstruction)
8986	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8987	}
8988	/* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */
8989
8990	return VINF_SUCCESS;
8991	}
8992
8993
8994	/**
8995	* Injects any pending events into the guest if the guest is in a state to
8996	* receive them.
8997	*
8998	* @returns Strict VBox status code (i.e. informational status codes too).
8999	* @param pVCpu The cross context virtual CPU structure.
9000	* @param pVmxTransient The VMX-transient structure.
9001	* @param fIntrState The VT-x guest-interruptibility state.
9002	* @param fStepping Whether we are single-stepping the guest using the
9003	* hypervisor debugger and should return
9004	* VINF_EM_DBG_STEPPED if the event was dispatched
9005	* directly.
9006	*/
9007	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9008	{
9009	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9010	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9011
9012	#ifdef VBOX_STRICT
9013	/*
9014	* Verify guest-interruptibility state.
9015	*
9016	* We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
9017	* since injecting an event may modify the interruptibility state and we must thus always
9018	* use fIntrState.
9019	*/
9020	{
9021	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9022	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9023	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9024	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9025	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9026	Assert(!TRPMHasTrap(pVCpu));
9027	NOREF(fBlockMovSS); NOREF(fBlockSti);
9028	}
9029	#endif
9030
9031	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9032	if (pVCpu->hm.s.Event.fPending)
9033	{
9034	/*
9035	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9036	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9037	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9038	*
9039	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9040	*/
9041	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9042	#ifdef VBOX_STRICT
9043	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9044	{
9045	Assert(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9046	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
9047	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9048	}
9049	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9050	{
9051	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
9052	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
9053	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9054	}
9055	#endif
9056	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9057	uIntType));
9058
9059	/*
9060	* Inject the event and get any changes to the guest-interruptibility state.
9061	*
9062	* The guest-interruptibility state may need to be updated if we inject the event
9063	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9064	*/
9065	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9066	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9067
9068	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9069	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9070	else
9071	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9072	}
9073
9074	/*
9075	* Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
9076	* is an interrupt shadow (block-by-STI or block-by-MOV SS).
9077	*/
9078	if ( (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9079	&& !pVmxTransient->fIsNestedGuest)
9080	{
9081	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9082
9083	if (!pVCpu->hm.s.fSingleInstruction)
9084	{
9085	/*
9086	* Set or clear the BS bit depending on whether the trap flag is active or not. We need
9087	* to do both since we clear the BS bit from the VMCS while exiting to ring-3.
9088	*/
9089	Assert(!DBGFIsStepping(pVCpu));
9090	uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_TF);
9091	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
9092	AssertRC(rc);
9093	}
9094	else
9095	{
9096	/*
9097	* We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
9098	* hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
9099	* we take care of this case in hmR0VmxExportSharedDebugState and also the case if
9100	* we use MTF, so just make sure it's called before executing guest-code.
9101	*/
9102	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
9103	}
9104	}
9105	/* else: for nested-guest currently handling while merging controls. */
9106
9107	/*
9108	* Finally, update the guest-interruptibility state.
9109	*
9110	* This is required for the real-on-v86 software interrupt injection, for
9111	* pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
9112	*/
9113	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9114	AssertRC(rc);
9115
9116	/*
9117	* There's no need to clear the VM-entry interruption-information field here if we're not
9118	* injecting anything. VT-x clears the valid bit on every VM-exit.
9119	*
9120	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9121	*/
9122
9123	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9124	return rcStrict;
9125	}
9126
9127
9128	/**
9129	* Enters the VT-x session.
9130	*
9131	* @returns VBox status code.
9132	* @param pVCpu The cross context virtual CPU structure.
9133	*/
9134	VMMR0DECL(int) VMXR0Enter(PVMCPUCC pVCpu)
9135	{
9136	AssertPtr(pVCpu);
9137	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9138	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9139
9140	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9141	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9142	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9143
9144	#ifdef VBOX_STRICT
9145	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9146	RTCCUINTREG uHostCr4 = ASMGetCR4();
9147	if (!(uHostCr4 & X86_CR4_VMXE))
9148	{
9149	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9150	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9151	}
9152	#endif
9153
9154	/*
9155	* Load the appropriate VMCS as the current and active one.
9156	*/
9157	PVMXVMCSINFO pVmcsInfo;
9158	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9159	if (!fInNestedGuestMode)
9160	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9161	else
9162	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9163	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9164	if (RT_SUCCESS(rc))
9165	{
9166	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9167	pVCpu->hm.s.fLeaveDone = false;
9168	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9169
9170	/*
9171	* Do the EMT scheduled L1D flush here if needed.
9172	*/
9173	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9174	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9175	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9176	hmR0MdsClear();
9177	}
9178	return rc;
9179	}
9180
9181
9182	/**
9183	* The thread-context callback (only on platforms which support it).
9184	*
9185	* @param enmEvent The thread-context event.
9186	* @param pVCpu The cross context virtual CPU structure.
9187	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9188	* @thread EMT(pVCpu)
9189	*/
9190	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
9191	{
9192	AssertPtr(pVCpu);
9193	RT_NOREF1(fGlobalInit);
9194
9195	switch (enmEvent)
9196	{
9197	case RTTHREADCTXEVENT_OUT:
9198	{
9199	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9200	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9201	VMCPU_ASSERT_EMT(pVCpu);
9202
9203	/* No longjmps (logger flushes, locks) in this fragile context. */
9204	VMMRZCallRing3Disable(pVCpu);
9205	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9206
9207	/* Restore host-state (FPU, debug etc.) */
9208	if (!pVCpu->hm.s.fLeaveDone)
9209	{
9210	/*
9211	* Do -not- import the guest-state here as we might already be in the middle of importing
9212	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9213	*/
9214	hmR0VmxLeave(pVCpu, false /* fImportState */);
9215	pVCpu->hm.s.fLeaveDone = true;
9216	}
9217
9218	/* Leave HM context, takes care of local init (term). */
9219	int rc = HMR0LeaveCpu(pVCpu);
9220	AssertRC(rc);
9221
9222	/* Restore longjmp state. */
9223	VMMRZCallRing3Enable(pVCpu);
9224	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9225	break;
9226	}
9227
9228	case RTTHREADCTXEVENT_IN:
9229	{
9230	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9231	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9232	VMCPU_ASSERT_EMT(pVCpu);
9233
9234	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9235	VMMRZCallRing3Disable(pVCpu);
9236	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9237
9238	/* Initialize the bare minimum state required for HM. This takes care of
9239	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9240	int rc = hmR0EnterCpu(pVCpu);
9241	AssertRC(rc);
9242	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9243	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9244
9245	/* Load the active VMCS as the current one. */
9246	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9247	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9248	AssertRC(rc);
9249	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9250	pVCpu->hm.s.fLeaveDone = false;
9251
9252	/* Do the EMT scheduled L1D flush if needed. */
9253	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9254	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9255
9256	/* Restore longjmp state. */
9257	VMMRZCallRing3Enable(pVCpu);
9258	break;
9259	}
9260
9261	default:
9262	break;
9263	}
9264	}
9265
9266
9267	/**
9268	* Exports the host state into the VMCS host-state area.
9269	* Sets up the VM-exit MSR-load area.
9270	*
9271	* The CPU state will be loaded from these fields on every successful VM-exit.
9272	*
9273	* @returns VBox status code.
9274	* @param pVCpu The cross context virtual CPU structure.
9275	*
9276	* @remarks No-long-jump zone!!!
9277	*/
9278	static int hmR0VmxExportHostState(PVMCPUCC pVCpu)
9279	{
9280	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9281
9282	int rc = VINF_SUCCESS;
9283	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9284	{
9285	hmR0VmxExportHostControlRegs();
9286
9287	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9288	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9289
9290	hmR0VmxExportHostMsrs(pVCpu);
9291
9292	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9293	}
9294	return rc;
9295	}
9296
9297
9298	/**
9299	* Saves the host state in the VMCS host-state.
9300	*
9301	* @returns VBox status code.
9302	* @param pVCpu The cross context virtual CPU structure.
9303	*
9304	* @remarks No-long-jump zone!!!
9305	*/
9306	VMMR0DECL(int) VMXR0ExportHostState(PVMCPUCC pVCpu)
9307	{
9308	AssertPtr(pVCpu);
9309	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9310
9311	/*
9312	* Export the host state here while entering HM context.
9313	* When thread-context hooks are used, we might get preempted and have to re-save the host
9314	* state but most of the time we won't be, so do it here before we disable interrupts.
9315	*/
9316	return hmR0VmxExportHostState(pVCpu);
9317	}
9318
9319
9320	/**
9321	* Exports the guest state into the VMCS guest-state area.
9322	*
9323	* The will typically be done before VM-entry when the guest-CPU state and the
9324	* VMCS state may potentially be out of sync.
9325	*
9326	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9327	* VM-entry controls.
9328	* Sets up the appropriate VMX non-root function to execute guest code based on
9329	* the guest CPU mode.
9330	*
9331	* @returns VBox strict status code.
9332	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9333	* without unrestricted guest execution and the VMMDev is not presently
9334	* mapped (e.g. EFI32).
9335	*
9336	* @param pVCpu The cross context virtual CPU structure.
9337	* @param pVmxTransient The VMX-transient structure.
9338	*
9339	* @remarks No-long-jump zone!!!
9340	*/
9341	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9342	{
9343	AssertPtr(pVCpu);
9344	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9345	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9346
9347	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9348
9349	/*
9350	* Determine real-on-v86 mode.
9351	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9352	*/
9353	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9354	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9355	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9356	pVmcsInfo->RealMode.fRealOnV86Active = false;
9357	else
9358	{
9359	Assert(!pVmxTransient->fIsNestedGuest);
9360	pVmcsInfo->RealMode.fRealOnV86Active = true;
9361	}
9362
9363	/*
9364	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9365	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9366	*/
9367	int rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9368	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9369
9370	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9371	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9372
9373	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9374	if (rcStrict == VINF_SUCCESS)
9375	{ /* likely */ }
9376	else
9377	{
9378	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9379	return rcStrict;
9380	}
9381
9382	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9383	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9384
9385	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9386	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9387
9388	hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9389	hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9390	hmR0VmxExportGuestRip(pVCpu);
9391	hmR0VmxExportGuestRsp(pVCpu);
9392	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9393
9394	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9395	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9396
9397	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9398	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9399	\| HM_CHANGED_GUEST_CR2
9400	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9401	\| HM_CHANGED_GUEST_X87
9402	\| HM_CHANGED_GUEST_SSE_AVX
9403	\| HM_CHANGED_GUEST_OTHER_XSAVE
9404	\| HM_CHANGED_GUEST_XCRx
9405	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9406	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9407	\| HM_CHANGED_GUEST_TSC_AUX
9408	\| HM_CHANGED_GUEST_OTHER_MSRS
9409	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9410
9411	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9412	return rc;
9413	}
9414
9415
9416	/**
9417	* Exports the state shared between the host and guest into the VMCS.
9418	*
9419	* @param pVCpu The cross context virtual CPU structure.
9420	* @param pVmxTransient The VMX-transient structure.
9421	*
9422	* @remarks No-long-jump zone!!!
9423	*/
9424	static void hmR0VmxExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9425	{
9426	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9427	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9428
9429	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9430	{
9431	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9432	AssertRC(rc);
9433	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9434
9435	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9436	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9437	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9438	}
9439
9440	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9441	{
9442	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9443	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9444	}
9445
9446	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9447	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9448	}
9449
9450
9451	/**
9452	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9453	*
9454	* @returns Strict VBox status code (i.e. informational status codes too).
9455	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9456	* without unrestricted guest execution and the VMMDev is not presently
9457	* mapped (e.g. EFI32).
9458	*
9459	* @param pVCpu The cross context virtual CPU structure.
9460	* @param pVmxTransient The VMX-transient structure.
9461	*
9462	* @remarks No-long-jump zone!!!
9463	*/
9464	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9465	{
9466	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9467	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9468	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9469
9470	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9471	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9472	#endif
9473
9474	/*
9475	* For many VM-exits only RIP/RSP/RFLAGS (and HWVIRT state when executing a nested-guest)
9476	* changes. First try to export only these without going through all other changed-flag checks.
9477	*/
9478	VBOXSTRICTRC rcStrict;
9479	uint64_t const fCtxMask = HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
9480	uint64_t const fMinimalMask = HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT;
9481	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9482
9483	/* If only RIP/RSP/RFLAGS/HWVIRT changed, export only those (quicker, happens more often).*/
9484	if ( (fCtxChanged & fMinimalMask)
9485	&& !(fCtxChanged & (fCtxMask & ~fMinimalMask)))
9486	{
9487	hmR0VmxExportGuestRip(pVCpu);
9488	hmR0VmxExportGuestRsp(pVCpu);
9489	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9490	rcStrict = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9491	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9492	}
9493	/* If anything else also changed, go through the full export routine and export as required. */
9494	else if (fCtxChanged & fCtxMask)
9495	{
9496	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9497	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9498	{ /* likely */}
9499	else
9500	{
9501	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9502	VBOXSTRICTRC_VAL(rcStrict)));
9503	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9504	return rcStrict;
9505	}
9506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9507	}
9508	/* Nothing changed, nothing to load here. */
9509	else
9510	rcStrict = VINF_SUCCESS;
9511
9512	#ifdef VBOX_STRICT
9513	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9514	uint64_t const fCtxChangedCur = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9515	AssertMsg(!(fCtxChangedCur & fCtxMask), ("fCtxChangedCur=%#RX64\n", fCtxChangedCur));
9516	#endif
9517	return rcStrict;
9518	}
9519
9520
9521	/**
9522	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9523	* and update error record fields accordingly.
9524	*
9525	* @returns VMX_IGS_* error codes.
9526	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9527	* wrong with the guest state.
9528	*
9529	* @param pVCpu The cross context virtual CPU structure.
9530	* @param pVmcsInfo The VMCS info. object.
9531	*
9532	* @remarks This function assumes our cache of the VMCS controls
9533	* are valid, i.e. hmR0VmxCheckCachedVmcsCtls() succeeded.
9534	*/
9535	static uint32_t hmR0VmxCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9536	{
9537	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9538	#define HMVMX_CHECK_BREAK(expr, err) do { \
9539	if (!(expr)) { uError = (err); break; } \
9540	} while (0)
9541
9542	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9543	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9544	uint32_t uError = VMX_IGS_ERROR;
9545	uint32_t u32IntrState = 0;
9546	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9547	do
9548	{
9549	int rc;
9550
9551	/*
9552	* Guest-interruptibility state.
9553	*
9554	* Read this first so that any check that fails prior to those that actually
9555	* require the guest-interruptibility state would still reflect the correct
9556	* VMCS value and avoids causing further confusion.
9557	*/
9558	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9559	AssertRC(rc);
9560
9561	uint32_t u32Val;
9562	uint64_t u64Val;
9563
9564	/*
9565	* CR0.
9566	*/
9567	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9568	uint64_t fSetCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9569	uint64_t const fZapCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9570	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
9571	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9572	if (fUnrestrictedGuest)
9573	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
9574
9575	uint64_t u64GuestCr0;
9576	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64GuestCr0);
9577	AssertRC(rc);
9578	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9579	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9580	if ( !fUnrestrictedGuest
9581	&& (u64GuestCr0 & X86_CR0_PG)
9582	&& !(u64GuestCr0 & X86_CR0_PE))
9583	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9584
9585	/*
9586	* CR4.
9587	*/
9588	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9589	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9590	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9591
9592	uint64_t u64GuestCr4;
9593	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64GuestCr4);
9594	AssertRC(rc);
9595	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9596	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9597
9598	/*
9599	* IA32_DEBUGCTL MSR.
9600	*/
9601	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9602	AssertRC(rc);
9603	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9604	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9605	{
9606	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9607	}
9608	uint64_t u64DebugCtlMsr = u64Val;
9609
9610	#ifdef VBOX_STRICT
9611	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9612	AssertRC(rc);
9613	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9614	#endif
9615	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9616
9617	/*
9618	* RIP and RFLAGS.
9619	*/
9620	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
9621	AssertRC(rc);
9622	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9623	if ( !fLongModeGuest
9624	\|\| !pCtx->cs.Attr.n.u1Long)
9625	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9626	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9627	* must be identical if the "IA-32e mode guest" VM-entry
9628	* control is 1 and CS.L is 1. No check applies if the
9629	* CPU supports 64 linear-address bits. */
9630
9631	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9632	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9633	AssertRC(rc);
9634	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9635	VMX_IGS_RFLAGS_RESERVED);
9636	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9637	uint32_t const u32Eflags = u64Val;
9638
9639	if ( fLongModeGuest
9640	\|\| ( fUnrestrictedGuest
9641	&& !(u64GuestCr0 & X86_CR0_PE)))
9642	{
9643	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9644	}
9645
9646	uint32_t u32EntryInfo;
9647	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9648	AssertRC(rc);
9649	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9650	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9651
9652	/*
9653	* 64-bit checks.
9654	*/
9655	if (fLongModeGuest)
9656	{
9657	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9658	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9659	}
9660
9661	if ( !fLongModeGuest
9662	&& (u64GuestCr4 & X86_CR4_PCIDE))
9663	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9664
9665	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9666	* 51:32 beyond the processor's physical-address width are 0. */
9667
9668	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9669	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9670	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9671
9672	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9673	AssertRC(rc);
9674	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9675
9676	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9677	AssertRC(rc);
9678	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9679
9680	/*
9681	* PERF_GLOBAL MSR.
9682	*/
9683	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9684	{
9685	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9686	AssertRC(rc);
9687	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9688	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9689	}
9690
9691	/*
9692	* PAT MSR.
9693	*/
9694	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9695	{
9696	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9697	AssertRC(rc);
9698	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9699	for (unsigned i = 0; i < 8; i++)
9700	{
9701	uint8_t u8Val = (u64Val & 0xff);
9702	if ( u8Val != 0 /* UC */
9703	&& u8Val != 1 /* WC */
9704	&& u8Val != 4 /* WT */
9705	&& u8Val != 5 /* WP */
9706	&& u8Val != 6 /* WB */
9707	&& u8Val != 7 /* UC- */)
9708	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9709	u64Val >>= 8;
9710	}
9711	}
9712
9713	/*
9714	* EFER MSR.
9715	*/
9716	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9717	{
9718	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9719	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9720	AssertRC(rc);
9721	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9722	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9723	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9724	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9725	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9726	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9727	* iemVmxVmentryCheckGuestState(). */
9728	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9729	\|\| !(u64GuestCr0 & X86_CR0_PG)
9730	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9731	VMX_IGS_EFER_LMA_LME_MISMATCH);
9732	}
9733
9734	/*
9735	* Segment registers.
9736	*/
9737	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9738	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9739	if (!(u32Eflags & X86_EFL_VM))
9740	{
9741	/* CS */
9742	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9743	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9744	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9745	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9746	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9747	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9748	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9749	/* CS cannot be loaded with NULL in protected mode. */
9750	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9751	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9752	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9753	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9754	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9755	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9756	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9757	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9758	else
9759	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9760
9761	/* SS */
9762	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9763	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9764	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9765	if ( !(pCtx->cr0 & X86_CR0_PE)
9766	\|\| pCtx->cs.Attr.n.u4Type == 3)
9767	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9768
9769	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9770	{
9771	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9772	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9773	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9774	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9775	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9776	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9777	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9778	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9779	}
9780
9781	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9782	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9783	{
9784	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9785	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9786	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9787	\|\| pCtx->ds.Attr.n.u4Type > 11
9788	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9789	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9790	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9791	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9792	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9793	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9794	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9795	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9796	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9797	}
9798	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9799	{
9800	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9801	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9802	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9803	\|\| pCtx->es.Attr.n.u4Type > 11
9804	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9805	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9806	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9807	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9808	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9809	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9810	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9811	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9812	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9813	}
9814	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9815	{
9816	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9817	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9818	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9819	\|\| pCtx->fs.Attr.n.u4Type > 11
9820	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9821	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9822	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9823	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9824	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9825	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9826	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9827	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9828	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9829	}
9830	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9831	{
9832	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9833	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9834	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9835	\|\| pCtx->gs.Attr.n.u4Type > 11
9836	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9837	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9838	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9839	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9840	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9841	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9842	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9843	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9844	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9845	}
9846	/* 64-bit capable CPUs. */
9847	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9848	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9849	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9850	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9851	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9852	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9853	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9854	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9855	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9856	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9857	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9858	}
9859	else
9860	{
9861	/* V86 mode checks. */
9862	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9863	if (pVmcsInfo->RealMode.fRealOnV86Active)
9864	{
9865	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9866	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9867	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9868	}
9869	else
9870	{
9871	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9872	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9873	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9874	}
9875
9876	/* CS */
9877	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9878	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9879	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9880	/* SS */
9881	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9882	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9883	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9884	/* DS */
9885	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9886	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9887	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9888	/* ES */
9889	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9890	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9891	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9892	/* FS */
9893	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9894	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9895	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9896	/* GS */
9897	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9898	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9899	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9900	/* 64-bit capable CPUs. */
9901	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9902	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9903	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9904	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9905	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9906	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9907	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9908	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9909	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9910	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9911	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9912	}
9913
9914	/*
9915	* TR.
9916	*/
9917	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9918	/* 64-bit capable CPUs. */
9919	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9920	if (fLongModeGuest)
9921	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9922	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9923	else
9924	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9925	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9926	VMX_IGS_TR_ATTR_TYPE_INVALID);
9927	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9928	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9929	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9930	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9931	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9932	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9933	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9934	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9935
9936	/*
9937	* GDTR and IDTR (64-bit capable checks).
9938	*/
9939	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9940	AssertRC(rc);
9941	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9942
9943	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9944	AssertRC(rc);
9945	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9946
9947	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9948	AssertRC(rc);
9949	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9950
9951	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9952	AssertRC(rc);
9953	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9954
9955	/*
9956	* Guest Non-Register State.
9957	*/
9958	/* Activity State. */
9959	uint32_t u32ActivityState;
9960	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9961	AssertRC(rc);
9962	HMVMX_CHECK_BREAK( !u32ActivityState
9963	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9964	VMX_IGS_ACTIVITY_STATE_INVALID);
9965	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9966	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9967
9968	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9969	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9970	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9971
9972	/** @todo Activity state and injecting interrupts. Left as a todo since we
9973	* currently don't use activity states but ACTIVE. */
9974
9975	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9976	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9977
9978	/* Guest interruptibility-state. */
9979	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9980	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9981	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9982	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9983	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9984	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9985	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9986	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9987	{
9988	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9989	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9990	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9991	}
9992	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9993	{
9994	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9995	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9996	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9997	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9998	}
9999	/** @todo Assumes the processor is not in SMM. */
10000	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10001	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10002	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10003	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10004	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10005	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10006	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
10007	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10008
10009	/* Pending debug exceptions. */
10010	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10011	AssertRC(rc);
10012	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10013	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10014	u32Val = u64Val; /* For pending debug exceptions checks below. */
10015
10016	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10017	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10018	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10019	{
10020	if ( (u32Eflags & X86_EFL_TF)
10021	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10022	{
10023	/* Bit 14 is PendingDebug.BS. */
10024	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10025	}
10026	if ( !(u32Eflags & X86_EFL_TF)
10027	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10028	{
10029	/* Bit 14 is PendingDebug.BS. */
10030	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10031	}
10032	}
10033
10034	/* VMCS link pointer. */
10035	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10036	AssertRC(rc);
10037	if (u64Val != UINT64_C(0xffffffffffffffff))
10038	{
10039	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10040	/** @todo Bits beyond the processor's physical-address width MBZ. */
10041	/** @todo SMM checks. */
10042	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10043	Assert(pVmcsInfo->pvShadowVmcs);
10044	VMXVMCSREVID VmcsRevId;
10045	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10046	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10047	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10048	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10049	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10050	}
10051
10052	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10053	* not using nested paging? */
10054	if ( pVM->hm.s.fNestedPaging
10055	&& !fLongModeGuest
10056	&& CPUMIsGuestInPAEModeEx(pCtx))
10057	{
10058	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10059	AssertRC(rc);
10060	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10061
10062	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10063	AssertRC(rc);
10064	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10065
10066	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10067	AssertRC(rc);
10068	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10069
10070	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10071	AssertRC(rc);
10072	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10073	}
10074
10075	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10076	if (uError == VMX_IGS_ERROR)
10077	uError = VMX_IGS_REASON_NOT_FOUND;
10078	} while (0);
10079
10080	pVCpu->hm.s.u32HMError = uError;
10081	pVCpu->hm.s.vmx.LastError.u32GuestIntrState = u32IntrState;
10082	return uError;
10083
10084	#undef HMVMX_ERROR_BREAK
10085	#undef HMVMX_CHECK_BREAK
10086	}
10087
10088
10089	/**
10090	* Map the APIC-access page for virtualizing APIC accesses.
10091	*
10092	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10093	* this not done as part of exporting guest state, see @bugref{8721}.
10094	*
10095	* @returns VBox status code.
10096	* @param pVCpu The cross context virtual CPU structure.
10097	*/
10098	static int hmR0VmxMapHCApicAccessPage(PVMCPUCC pVCpu)
10099	{
10100	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10101	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10102
10103	Assert(PDMHasApic(pVM));
10104	Assert(u64MsrApicBase);
10105
10106	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10107	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10108
10109	/* Unalias the existing mapping. */
10110	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10111	AssertRCReturn(rc, rc);
10112
10113	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10114	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10115	rc = IOMR0MmioMapMmioHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10116	AssertRCReturn(rc, rc);
10117
10118	/* Update the per-VCPU cache of the APIC base MSR. */
10119	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10120	return VINF_SUCCESS;
10121	}
10122
10123
10124	/**
10125	* Worker function passed to RTMpOnSpecific() that is to be called on the target
10126	* CPU.
10127	*
10128	* @param idCpu The ID for the CPU the function is called on.
10129	* @param pvUser1 Null, not used.
10130	* @param pvUser2 Null, not used.
10131	*/
10132	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
10133	{
10134	RT_NOREF3(idCpu, pvUser1, pvUser2);
10135	VMXDispatchHostNmi();
10136	}
10137
10138
10139	/**
10140	* Dispatching an NMI on the host CPU that received it.
10141	*
10142	* @returns VBox status code.
10143	* @param pVCpu The cross context virtual CPU structure.
10144	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
10145	* executing when receiving the host NMI in VMX non-root
10146	* operation.
10147	*/
10148	static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
10149	{
10150	RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
10151	Assert(idCpu != NIL_RTCPUID);
10152
10153	/*
10154	* We don't want to delay dispatching the NMI any more than we have to. However,
10155	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
10156	* after executing guest or nested-guest code for the following reasons:
10157	*
10158	* - We would need to perform VMREADs with interrupts disabled and is orders of
10159	* magnitude worse when we run as a nested hypervisor without VMCS shadowing
10160	* supported by the host hypervisor.
10161	*
10162	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
10163	* longer period of time just for handling an edge case like host NMIs which do
10164	* not occur nearly as frequently as other VM-exits.
10165	*
10166	* Let's cover the most likely scenario first. Check if we are on the target CPU
10167	* and dispatch the NMI right away. This should be much faster than calling into
10168	* RTMpOnSpecific() machinery.
10169	*/
10170	bool fDispatched = false;
10171	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10172	if (idCpu == RTMpCpuId())
10173	{
10174	VMXDispatchHostNmi();
10175	fDispatched = true;
10176	}
10177	ASMSetFlags(fEFlags);
10178	if (fDispatched)
10179	{
10180	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10181	return VINF_SUCCESS;
10182	}
10183
10184	/*
10185	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
10186	* there should be no race or recursion even if we are unlucky enough to be preempted
10187	* (to the target CPU) without dispatching the host NMI above.
10188	*/
10189	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
10190	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
10191	}
10192
10193
10194	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10195	/**
10196	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10197	* nested-guest using hardware-assisted VMX.
10198	*
10199	* @param pVCpu The cross context virtual CPU structure.
10200	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10201	* @param pVmcsInfoGst The guest VMCS info. object.
10202	*/
10203	static void hmR0VmxMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10204	{
10205	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10206	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10207	Assert(pu64MsrBitmap);
10208
10209	/*
10210	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10211	* MSR that is intercepted by the guest is also intercepted while executing the
10212	* nested-guest using hardware-assisted VMX.
10213	*
10214	* Note! If the nested-guest is not using an MSR bitmap, every MSR must cause a
10215	* nested-guest VM-exit even if the outer guest is not intercepting some
10216	* MSRs. We cannot assume the caller has initialized the nested-guest
10217	* MSR bitmap in this case.
10218	*
10219	* The nested hypervisor may also switch whether it uses MSR bitmaps for
10220	* each of its VM-entry, hence initializing it once per-VM while setting
10221	* up the nested-guest VMCS is not sufficient.
10222	*/
10223	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10224	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10225	{
10226	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10227	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10228	Assert(pu64MsrBitmapNstGst);
10229	Assert(pu64MsrBitmapGst);
10230
10231	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10232	for (uint32_t i = 0; i < cFrags; i++)
10233	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10234	}
10235	else
10236	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10237	}
10238
10239
10240	/**
10241	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10242	* hardware-assisted VMX execution of the nested-guest.
10243	*
10244	* For a guest, we don't modify these controls once we set up the VMCS and hence
10245	* this function is never called.
10246	*
10247	* For nested-guests since the nested hypervisor provides these controls on every
10248	* nested-guest VM-entry and could potentially change them everytime we need to
10249	* merge them before every nested-guest VM-entry.
10250	*
10251	* @returns VBox status code.
10252	* @param pVCpu The cross context virtual CPU structure.
10253	*/
10254	static int hmR0VmxMergeVmcsNested(PVMCPUCC pVCpu)
10255	{
10256	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10257	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10258	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10259	Assert(pVmcsNstGst);
10260
10261	/*
10262	* Merge the controls with the requirements of the guest VMCS.
10263	*
10264	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10265	* VMCS with the features supported by the physical CPU as it's already done by the
10266	* VMLAUNCH/VMRESUME instruction emulation.
10267	*
10268	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10269	* derived from the VMX features supported by the physical CPU.
10270	*/
10271
10272	/* Pin-based VM-execution controls. */
10273	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10274
10275	/* Processor-based VM-execution controls. */
10276	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10277	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10278	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10279	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10280	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10281
10282	/* Secondary processor-based VM-execution controls. */
10283	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10284	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10285	\| VMX_PROC_CTLS2_INVPCID
10286	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10287	\| VMX_PROC_CTLS2_RDTSCP
10288	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10289	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10290	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10291	\| VMX_PROC_CTLS2_VMFUNC));
10292
10293	/*
10294	* VM-entry controls:
10295	* These controls contains state that depends on the nested-guest state (primarily
10296	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10297	* VM-exit. Although the nested hypervisor cannot change it, we need to in order to
10298	* properly continue executing the nested-guest if the EFER MSR changes but does not
10299	* cause a nested-guest VM-exits.
10300	*
10301	* VM-exit controls:
10302	* These controls specify the host state on return. We cannot use the controls from
10303	* the nested hypervisor state as is as it would contain the guest state rather than
10304	* the host state. Since the host state is subject to change (e.g. preemption, trips
10305	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10306	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10307	*
10308	* VM-entry MSR-load:
10309	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10310	* context by the VMLAUNCH/VMRESUME instruction emulation.
10311	*
10312	* VM-exit MSR-store:
10313	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10314	* back into the VM-exit MSR-store area.
10315	*
10316	* VM-exit MSR-load areas:
10317	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10318	* can entirely ignore what the nested hypervisor wants to load here.
10319	*/
10320
10321	/*
10322	* Exception bitmap.
10323	*
10324	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10325	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10326	* code more flexible if intercepting exceptions become more dynamic in the future we do
10327	* it as part of exporting the nested-guest state.
10328	*/
10329	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10330
10331	/*
10332	* CR0/CR4 guest/host mask.
10333	*
10334	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10335	* cause VM-exits, so we need to merge them here.
10336	*/
10337	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10338	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10339
10340	/*
10341	* Page-fault error-code mask and match.
10342	*
10343	* Although we require unrestricted guest execution (and thereby nested-paging) for
10344	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10345	* normally intercept #PFs, it might intercept them for debugging purposes.
10346	*
10347	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10348	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10349	*/
10350	uint32_t u32XcptPFMask;
10351	uint32_t u32XcptPFMatch;
10352	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10353	{
10354	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10355	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10356	}
10357	else
10358	{
10359	u32XcptPFMask = 0;
10360	u32XcptPFMatch = 0;
10361	}
10362
10363	/*
10364	* Pause-Loop exiting.
10365	*/
10366	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10367	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10368
10369	/*
10370	* Pending debug exceptions.
10371	* Currently just copy whatever the nested-guest provides us.
10372	*/
10373	uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
10374
10375	/*
10376	* I/O Bitmap.
10377	*
10378	* We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
10379	* intercept all I/O port accesses.
10380	*/
10381	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10382	Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
10383
10384	/*
10385	* VMCS shadowing.
10386	*
10387	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10388	* enabled while executing the nested-guest.
10389	*/
10390	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10391
10392	/*
10393	* APIC-access page.
10394	*/
10395	RTHCPHYS HCPhysApicAccess;
10396	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10397	{
10398	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10399	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10400
10401	/** @todo NSTVMX: This is not really correct but currently is required to make
10402	* things work. We need to re-enable the page handler when we fallback to
10403	* IEM execution of the nested-guest! */
10404	PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
10405
10406	void *pvPage;
10407	PGMPAGEMAPLOCK PgLockApicAccess;
10408	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
10409	if (RT_SUCCESS(rc))
10410	{
10411	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10412	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10413
10414	/** @todo Handle proper releasing of page-mapping lock later. */
10415	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
10416	}
10417	else
10418	return rc;
10419	}
10420	else
10421	HCPhysApicAccess = 0;
10422
10423	/*
10424	* Virtual-APIC page and TPR threshold.
10425	*/
10426	RTHCPHYS HCPhysVirtApic;
10427	uint32_t u32TprThreshold;
10428	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10429	{
10430	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10431	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10432
10433	void *pvPage;
10434	PGMPAGEMAPLOCK PgLockVirtApic;
10435	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
10436	if (RT_SUCCESS(rc))
10437	{
10438	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10439	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10440
10441	/** @todo Handle proper releasing of page-mapping lock later. */
10442	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
10443	}
10444	else
10445	return rc;
10446
10447	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10448	}
10449	else
10450	{
10451	HCPhysVirtApic = 0;
10452	u32TprThreshold = 0;
10453
10454	/*
10455	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10456	* used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
10457	* be taken care of by EPT/shadow paging.
10458	*/
10459	if (pVM->hm.s.fAllow64BitGuests)
10460	{
10461	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10462	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10463	}
10464	}
10465
10466	/*
10467	* Validate basic assumptions.
10468	*/
10469	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10470	Assert(pVM->hm.s.vmx.fUnrestrictedGuest);
10471	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10472	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10473
10474	/*
10475	* Commit it to the nested-guest VMCS.
10476	*/
10477	int rc = VINF_SUCCESS;
10478	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10479	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10480	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10481	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10482	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10483	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10484	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10485	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10486	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10487	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10488	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10489	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10490	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10491	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10492	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10493	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10494	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10495	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10496	{
10497	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10498	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10499	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10500	}
10501	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10502	{
10503	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10504	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10505	}
10506	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10507	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10508	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
10509	AssertRC(rc);
10510
10511	/*
10512	* Update the nested-guest VMCS cache.
10513	*/
10514	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10515	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10516	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10517	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10518	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10519	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10520	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10521	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10522	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10523
10524	/*
10525	* We need to flush the TLB if we are switching the APIC-access page address.
10526	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10527	*/
10528	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10529	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10530
10531	/*
10532	* MSR bitmap.
10533	*
10534	* The MSR bitmap address has already been initialized while setting up the nested-guest
10535	* VMCS, here we need to merge the MSR bitmaps.
10536	*/
10537	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10538	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10539
10540	return VINF_SUCCESS;
10541	}
10542	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10543
10544
10545	/**
10546	* Does the preparations before executing guest code in VT-x.
10547	*
10548	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10549	* recompiler/IEM. We must be cautious what we do here regarding committing
10550	* guest-state information into the VMCS assuming we assuredly execute the
10551	* guest in VT-x mode.
10552	*
10553	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10554	* the common-state (TRPM/forceflags), we must undo those changes so that the
10555	* recompiler/IEM can (and should) use them when it resumes guest execution.
10556	* Otherwise such operations must be done when we can no longer exit to ring-3.
10557	*
10558	* @returns Strict VBox status code (i.e. informational status codes too).
10559	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10560	* have been disabled.
10561	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10562	* pending events).
10563	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10564	* double-fault into the guest.
10565	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10566	* dispatched directly.
10567	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10568	*
10569	* @param pVCpu The cross context virtual CPU structure.
10570	* @param pVmxTransient The VMX-transient structure.
10571	* @param fStepping Whether we are single-stepping the guest in the
10572	* hypervisor debugger. Makes us ignore some of the reasons
10573	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10574	* if event dispatching took place.
10575	*/
10576	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10577	{
10578	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10579
10580	Log4Func(("fIsNested=%RTbool fStepping=%RTbool\n", pVmxTransient->fIsNestedGuest, fStepping));
10581
10582	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10583	if (pVmxTransient->fIsNestedGuest)
10584	{
10585	RT_NOREF2(pVCpu, fStepping);
10586	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10587	return VINF_EM_RESCHEDULE_REM;
10588	}
10589	#endif
10590
10591	/*
10592	* Check and process force flag actions, some of which might require us to go back to ring-3.
10593	*/
10594	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pVmxTransient, fStepping);
10595	if (rcStrict == VINF_SUCCESS)
10596	{
10597	/* FFs don't get set all the time. */
10598	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10599	if ( pVmxTransient->fIsNestedGuest
10600	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10601	{
10602	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10603	return VINF_VMX_VMEXIT;
10604	}
10605	#endif
10606	}
10607	else
10608	return rcStrict;
10609
10610	/*
10611	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10612	*/
10613	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10614	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10615	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10616	&& PDMHasApic(pVM))
10617	{
10618	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10619	AssertRCReturn(rc, rc);
10620	}
10621
10622	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10623	/*
10624	* Merge guest VMCS controls with the nested-guest VMCS controls.
10625	*
10626	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10627	* saved state), we should be okay with merging controls as we initialize the
10628	* guest VMCS controls as part of VM setup phase.
10629	*/
10630	if ( pVmxTransient->fIsNestedGuest
10631	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10632	{
10633	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10634	AssertRCReturn(rc, rc);
10635	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10636	}
10637	#endif
10638
10639	/*
10640	* Evaluate events to be injected into the guest.
10641	*
10642	* Events in TRPM can be injected without inspecting the guest state.
10643	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10644	* guest to cause a VM-exit the next time they are ready to receive the event.
10645	*
10646	* With nested-guests, evaluating pending events may cause VM-exits. Also, verify
10647	* that the event in TRPM that we will inject using hardware-assisted VMX is -not-
10648	* subject to interecption. Otherwise, we should have checked and injected them
10649	* manually elsewhere (IEM).
10650	*/
10651	if (TRPMHasTrap(pVCpu))
10652	{
10653	Assert(!pVmxTransient->fIsNestedGuest \|\| !CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
10654	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10655	}
10656
10657	uint32_t fIntrState;
10658	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10659
10660	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10661	/*
10662	* While evaluating pending events if something failed (unlikely) or if we were
10663	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10664	*/
10665	if (rcStrict != VINF_SUCCESS)
10666	return rcStrict;
10667	if ( pVmxTransient->fIsNestedGuest
10668	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10669	{
10670	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10671	return VINF_VMX_VMEXIT;
10672	}
10673	#else
10674	Assert(rcStrict == VINF_SUCCESS);
10675	#endif
10676
10677	/*
10678	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10679	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10680	* also result in triple-faulting the VM.
10681	*
10682	* With nested-guests, the above does not apply since unrestricted guest execution is a
10683	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10684	*/
10685	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10686	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10687	{ /* likely */ }
10688	else
10689	{
10690	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10691	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10692	return rcStrict;
10693	}
10694
10695	/*
10696	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10697	* import CR3 themselves. We will need to update them here, as even as late as the above
10698	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10699	* the below force flags to be set.
10700	*/
10701	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10702	{
10703	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10704	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10705	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10706	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10707	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10708	}
10709	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10710	{
10711	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10712	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10713	}
10714
10715	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10716	/* Paranoia. */
10717	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10718	#endif
10719
10720	/*
10721	* No longjmps to ring-3 from this point on!!!
10722	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10723	* This also disables flushing of the R0-logger instance (if any).
10724	*/
10725	VMMRZCallRing3Disable(pVCpu);
10726
10727	/*
10728	* Export the guest state bits.
10729	*
10730	* We cannot perform longjmps while loading the guest state because we do not preserve the
10731	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10732	* CPU migration.
10733	*
10734	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10735	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10736	*/
10737	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10738	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10739	{ /* likely */ }
10740	else
10741	{
10742	VMMRZCallRing3Enable(pVCpu);
10743	return rcStrict;
10744	}
10745
10746	/*
10747	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10748	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10749	* preemption disabled for a while. Since this is purely to aid the
10750	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10751	* disable interrupt on NT.
10752	*
10753	* We need to check for force-flags that could've possible been altered since we last
10754	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10755	* see @bugref{6398}).
10756	*
10757	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10758	* to ring-3 before executing guest code.
10759	*/
10760	pVmxTransient->fEFlags = ASMIntDisableFlags();
10761
10762	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10763	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10764	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10765	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10766	{
10767	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10768	{
10769	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10770	/*
10771	* If we are executing a nested-guest make sure that we should intercept subsequent
10772	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10773	* the VM-exit instruction emulation happy.
10774	*/
10775	if (pVmxTransient->fIsNestedGuest)
10776	CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
10777	#endif
10778
10779	/*
10780	* We've injected any pending events. This is really the point of no return (to ring-3).
10781	*
10782	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10783	* returns from this function, so do -not- enable them here.
10784	*/
10785	pVCpu->hm.s.Event.fPending = false;
10786	return VINF_SUCCESS;
10787	}
10788
10789	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10790	rcStrict = VINF_EM_RAW_INTERRUPT;
10791	}
10792	else
10793	{
10794	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10795	rcStrict = VINF_EM_RAW_TO_R3;
10796	}
10797
10798	ASMSetFlags(pVmxTransient->fEFlags);
10799	VMMRZCallRing3Enable(pVCpu);
10800
10801	return rcStrict;
10802	}
10803
10804
10805	/**
10806	* Final preparations before executing guest code using hardware-assisted VMX.
10807	*
10808	* We can no longer get preempted to a different host CPU and there are no returns
10809	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10810	* failures), this function is not intended to fail sans unrecoverable hardware
10811	* errors.
10812	*
10813	* @param pVCpu The cross context virtual CPU structure.
10814	* @param pVmxTransient The VMX-transient structure.
10815	*
10816	* @remarks Called with preemption disabled.
10817	* @remarks No-long-jump zone!!!
10818	*/
10819	static void hmR0VmxPreRunGuestCommitted(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10820	{
10821	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10822	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10823	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10824	Assert(!pVCpu->hm.s.Event.fPending);
10825
10826	/*
10827	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10828	*/
10829	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10830	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10831
10832	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10833	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10834	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10835	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10836
10837	if (!CPUMIsGuestFPUStateActive(pVCpu))
10838	{
10839	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10840	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10841	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10842	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10843	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10844	}
10845
10846	/*
10847	* Re-export the host state bits as we may've been preempted (only happens when
10848	* thread-context hooks are used or when the VM start function changes) or if
10849	* the host CR0 is modified while loading the guest FPU state above.
10850	*
10851	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10852	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10853	* see @bugref{8432}.
10854	*
10855	* This may also happen when switching to/from a nested-guest VMCS without leaving
10856	* ring-0.
10857	*/
10858	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10859	{
10860	hmR0VmxExportHostState(pVCpu);
10861	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10862	}
10863	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10864
10865	/*
10866	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10867	*/
10868	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10869	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10870	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10871
10872	/*
10873	* Store status of the shared guest/host debug state at the time of VM-entry.
10874	*/
10875	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10876	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10877
10878	/*
10879	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10880	* more than one conditional check. The post-run side of our code shall determine
10881	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10882	*/
10883	if (pVmcsInfo->pbVirtApic)
10884	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10885
10886	/*
10887	* Update the host MSRs values in the VM-exit MSR-load area.
10888	*/
10889	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10890	{
10891	if (pVmcsInfo->cExitMsrLoad > 0)
10892	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10893	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10894	}
10895
10896	/*
10897	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10898	* VMX-preemption timer based on the next virtual sync clock deadline.
10899	*/
10900	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10901	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10902	{
10903	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10904	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10905	}
10906
10907	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10908	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10909	if (!fIsRdtscIntercepted)
10910	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10911	else
10912	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10913
10914	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10915	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10916	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10917	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Record the error reporting info. with the current host CPU. */
10918	pVmcsInfo->idHostCpuState = idCurrentCpu; /* Record the CPU for which the host-state has been exported. */
10919	pVmcsInfo->idHostCpuExec = idCurrentCpu; /* Record the CPU on which we shall execute. */
10920
10921	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10922
10923	TMNotifyStartOfExecution(pVM, pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10924	as we're about to start executing the guest. */
10925
10926	/*
10927	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10928	*
10929	* This is done this late as updating the TSC offsetting/preemption timer above
10930	* figures out if we can skip intercepting RDTSCP by calculating the number of
10931	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10932	*/
10933	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10934	&& !fIsRdtscIntercepted)
10935	{
10936	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10937
10938	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10939	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10940	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10941	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10942	AssertRC(rc);
10943	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10944	pVmxTransient->fRemoveTscAuxMsr = true;
10945	}
10946
10947	#ifdef VBOX_STRICT
10948	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10949	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10950	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10951	AssertRC(hmR0VmxCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10952	#endif
10953
10954	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10955	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10956	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10957	* see @bugref{9180#c54}. */
10958	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10959	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10960	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10961	#endif
10962	}
10963
10964
10965	/**
10966	* First C routine invoked after running guest code using hardware-assisted VMX.
10967	*
10968	* @param pVCpu The cross context virtual CPU structure.
10969	* @param pVmxTransient The VMX-transient structure.
10970	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10971	*
10972	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10973	*
10974	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10975	* unconditionally when it is safe to do so.
10976	*/
10977	static void hmR0VmxPostRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10978	{
10979	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10980
10981	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10982	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10983	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10984	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10985	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10986	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10987
10988	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10989	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10990	{
10991	uint64_t uGstTsc;
10992	if (!pVmxTransient->fIsNestedGuest)
10993	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10994	else
10995	{
10996	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10997	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10998	}
10999	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
11000	}
11001
11002	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11003	TMNotifyEndOfExecution(pVCpu->CTX_SUFF(pVM), pVCpu); /* Notify TM that the guest is no longer running. */
11004	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11005
11006	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11007	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11008	#ifdef VBOX_STRICT
11009	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11010	#endif
11011	Assert(!ASMIntAreEnabled());
11012	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11013	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11014
11015	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
11016	/*
11017	* Clean all the VMCS fields in the transient structure before reading
11018	* anything from the VMCS.
11019	*/
11020	pVmxTransient->uExitReason = 0;
11021	pVmxTransient->uExitIntErrorCode = 0;
11022	pVmxTransient->uExitQual = 0;
11023	pVmxTransient->uGuestLinearAddr = 0;
11024	pVmxTransient->uExitIntInfo = 0;
11025	pVmxTransient->cbExitInstr = 0;
11026	pVmxTransient->ExitInstrInfo.u = 0;
11027	pVmxTransient->uEntryIntInfo = 0;
11028	pVmxTransient->uEntryXcptErrorCode = 0;
11029	pVmxTransient->cbEntryInstr = 0;
11030	pVmxTransient->uIdtVectoringInfo = 0;
11031	pVmxTransient->uIdtVectoringErrorCode = 0;
11032	#endif
11033
11034	/*
11035	* Save the basic VM-exit reason and check if the VM-entry failed.
11036	* See Intel spec. 24.9.1 "Basic VM-exit Information".
11037	*/
11038	uint32_t uExitReason;
11039	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11040	AssertRC(rc);
11041	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11042	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11043
11044	/*
11045	* Log the VM-exit before logging anything else as otherwise it might be a
11046	* tad confusing what happens before and after the world-switch.
11047	*/
11048	HMVMX_LOG_EXIT(pVCpu, uExitReason);
11049
11050	/*
11051	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11052	* bitmap permissions, if it was added before VM-entry.
11053	*/
11054	if (pVmxTransient->fRemoveTscAuxMsr)
11055	{
11056	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11057	pVmxTransient->fRemoveTscAuxMsr = false;
11058	}
11059
11060	/*
11061	* Check if VMLAUNCH/VMRESUME succeeded.
11062	* If this failed, we cause a guru meditation and cease further execution.
11063	*
11064	* However, if we are executing a nested-guest we might fail if we use the
11065	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11066	*/
11067	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11068	{
11069	/*
11070	* Update the VM-exit history array here even if the VM-entry failed due to:
11071	* - Invalid guest state.
11072	* - MSR loading.
11073	* - Machine-check event.
11074	*
11075	* In any of the above cases we will still have a "valid" VM-exit reason
11076	* despite @a fVMEntryFailed being false.
11077	*
11078	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11079	*
11080	* Note! We don't have CS or RIP at this point. Will probably address that later
11081	* by amending the history entry added here.
11082	*/
11083	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11084	UINT64_MAX, uHostTsc);
11085
11086	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11087	{
11088	VMMRZCallRing3Enable(pVCpu);
11089
11090	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11091	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11092
11093	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
11094	hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
11095	#endif
11096
11097	/*
11098	* Import the guest-interruptibility state always as we need it while evaluating
11099	* injecting events on re-entry.
11100	*
11101	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11102	* checking for real-mode while exporting the state because all bits that cause
11103	* mode changes wrt CR0 are intercepted.
11104	*/
11105	uint64_t const fImportMask = CPUMCTX_EXTRN_HM_VMX_INT_STATE
11106	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11107	\| HMVMX_CPUMCTX_EXTRN_ALL
11108	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11109	\| CPUMCTX_EXTRN_RFLAGS
11110	#endif
11111	;
11112	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImportMask);
11113	AssertRC(rc);
11114
11115	/*
11116	* Sync the TPR shadow with our APIC state.
11117	*/
11118	if ( !pVmxTransient->fIsNestedGuest
11119	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
11120	{
11121	Assert(pVmcsInfo->pbVirtApic);
11122	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11123	{
11124	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11125	AssertRC(rc);
11126	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11127	}
11128	}
11129
11130	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11131	return;
11132	}
11133	}
11134	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11135	else if (pVmxTransient->fIsNestedGuest)
11136	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11137	#endif
11138	else
11139	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11140
11141	VMMRZCallRing3Enable(pVCpu);
11142	}
11143
11144
11145	/**
11146	* Runs the guest code using hardware-assisted VMX the normal way.
11147	*
11148	* @returns VBox status code.
11149	* @param pVCpu The cross context virtual CPU structure.
11150	* @param pcLoops Pointer to the number of executed loops.
11151	*/
11152	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
11153	{
11154	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11155	Assert(pcLoops);
11156	Assert(*pcLoops <= cMaxResumeLoops);
11157	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11158
11159	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11160	/*
11161	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11162	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11163	* guest VMCS while entering the VMX ring-0 session.
11164	*/
11165	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11166	{
11167	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11168	if (RT_SUCCESS(rc))
11169	{ /* likely */ }
11170	else
11171	{
11172	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11173	return rc;
11174	}
11175	}
11176	#endif
11177
11178	VMXTRANSIENT VmxTransient;
11179	RT_ZERO(VmxTransient);
11180	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11181
11182	/* Paranoia. */
11183	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11184
11185	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11186	for (;;)
11187	{
11188	Assert(!HMR0SuspendPending());
11189	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11190	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11191
11192	/*
11193	* Preparatory work for running nested-guest code, this may force us to
11194	* return to ring-3.
11195	*
11196	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11197	*/
11198	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11199	if (rcStrict != VINF_SUCCESS)
11200	break;
11201
11202	/* Interrupts are disabled at this point! */
11203	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11204	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11205	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11206	/* Interrupts are re-enabled at this point! */
11207
11208	/*
11209	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11210	*/
11211	if (RT_SUCCESS(rcRun))
11212	{ /* very likely */ }
11213	else
11214	{
11215	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11216	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11217	return rcRun;
11218	}
11219
11220	/*
11221	* Profile the VM-exit.
11222	*/
11223	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11224	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11225	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11226	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11227	HMVMX_START_EXIT_DISPATCH_PROF();
11228
11229	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11230
11231	/*
11232	* Handle the VM-exit.
11233	*/
11234	#ifdef HMVMX_USE_FUNCTION_TABLE
11235	rcStrict = g_aVMExitHandlers[VmxTransient.uExitReason].pfn(pVCpu, &VmxTransient);
11236	#else
11237	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11238	#endif
11239	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11240	if (rcStrict == VINF_SUCCESS)
11241	{
11242	if (++(*pcLoops) <= cMaxResumeLoops)
11243	continue;
11244	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11245	rcStrict = VINF_EM_RAW_INTERRUPT;
11246	}
11247	break;
11248	}
11249
11250	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11251	return rcStrict;
11252	}
11253
11254
11255	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11256	/**
11257	* Runs the nested-guest code using hardware-assisted VMX.
11258	*
11259	* @returns VBox status code.
11260	* @param pVCpu The cross context virtual CPU structure.
11261	* @param pcLoops Pointer to the number of executed loops.
11262	*
11263	* @sa hmR0VmxRunGuestCodeNormal.
11264	*/
11265	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
11266	{
11267	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11268	Assert(pcLoops);
11269	Assert(*pcLoops <= cMaxResumeLoops);
11270	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11271
11272	/*
11273	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11274	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11275	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11276	*/
11277	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11278	{
11279	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11280	if (RT_SUCCESS(rc))
11281	{ /* likely */ }
11282	else
11283	{
11284	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11285	return rc;
11286	}
11287	}
11288
11289	VMXTRANSIENT VmxTransient;
11290	RT_ZERO(VmxTransient);
11291	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11292	VmxTransient.fIsNestedGuest = true;
11293
11294	/* Paranoia. */
11295	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11296
11297	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11298	for (;;)
11299	{
11300	Assert(!HMR0SuspendPending());
11301	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11302	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11303
11304	/*
11305	* Preparatory work for running guest code, this may force us to
11306	* return to ring-3.
11307	*
11308	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11309	*/
11310	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11311	if (rcStrict != VINF_SUCCESS)
11312	break;
11313
11314	/* Interrupts are disabled at this point! */
11315	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11316	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11317	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11318	/* Interrupts are re-enabled at this point! */
11319
11320	/*
11321	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11322	*/
11323	if (RT_SUCCESS(rcRun))
11324	{ /* very likely */ }
11325	else
11326	{
11327	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11328	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11329	return rcRun;
11330	}
11331
11332	/*
11333	* Profile the VM-exit.
11334	*/
11335	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11337	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11338	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11339	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11340	HMVMX_START_EXIT_DISPATCH_PROF();
11341
11342	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11343
11344	/*
11345	* Handle the VM-exit.
11346	*/
11347	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11348	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11349	if (rcStrict == VINF_SUCCESS)
11350	{
11351	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11352	{
11353	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11354	rcStrict = VINF_VMX_VMEXIT;
11355	}
11356	else
11357	{
11358	if (++(*pcLoops) <= cMaxResumeLoops)
11359	continue;
11360	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11361	rcStrict = VINF_EM_RAW_INTERRUPT;
11362	}
11363	}
11364	else
11365	Assert(rcStrict != VINF_VMX_VMEXIT);
11366	break;
11367	}
11368
11369	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11370	return rcStrict;
11371	}
11372	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11373
11374
11375	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11376	* probes.
11377	*
11378	* The following few functions and associated structure contains the bloat
11379	* necessary for providing detailed debug events and dtrace probes as well as
11380	* reliable host side single stepping. This works on the principle of
11381	* "subclassing" the normal execution loop and workers. We replace the loop
11382	* method completely and override selected helpers to add necessary adjustments
11383	* to their core operation.
11384	*
11385	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11386	* any performance for debug and analysis features.
11387	*
11388	* @{
11389	*/
11390
11391	/**
11392	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11393	* the debug run loop.
11394	*/
11395	typedef struct VMXRUNDBGSTATE
11396	{
11397	/** The RIP we started executing at. This is for detecting that we stepped. */
11398	uint64_t uRipStart;
11399	/** The CS we started executing with. */
11400	uint16_t uCsStart;
11401
11402	/** Whether we've actually modified the 1st execution control field. */
11403	bool fModifiedProcCtls : 1;
11404	/** Whether we've actually modified the 2nd execution control field. */
11405	bool fModifiedProcCtls2 : 1;
11406	/** Whether we've actually modified the exception bitmap. */
11407	bool fModifiedXcptBitmap : 1;
11408
11409	/** We desire the modified the CR0 mask to be cleared. */
11410	bool fClearCr0Mask : 1;
11411	/** We desire the modified the CR4 mask to be cleared. */
11412	bool fClearCr4Mask : 1;
11413	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11414	uint32_t fCpe1Extra;
11415	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11416	uint32_t fCpe1Unwanted;
11417	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11418	uint32_t fCpe2Extra;
11419	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11420	uint32_t bmXcptExtra;
11421	/** The sequence number of the Dtrace provider settings the state was
11422	* configured against. */
11423	uint32_t uDtraceSettingsSeqNo;
11424	/** VM-exits to check (one bit per VM-exit). */
11425	uint32_t bmExitsToCheck[3];
11426
11427	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11428	uint32_t fProcCtlsInitial;
11429	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11430	uint32_t fProcCtls2Initial;
11431	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11432	uint32_t bmXcptInitial;
11433	} VMXRUNDBGSTATE;
11434	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11435	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11436
11437
11438	/**
11439	* Initializes the VMXRUNDBGSTATE structure.
11440	*
11441	* @param pVCpu The cross context virtual CPU structure of the
11442	* calling EMT.
11443	* @param pVmxTransient The VMX-transient structure.
11444	* @param pDbgState The debug state to initialize.
11445	*/
11446	static void hmR0VmxRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11447	{
11448	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11449	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11450
11451	pDbgState->fModifiedProcCtls = false;
11452	pDbgState->fModifiedProcCtls2 = false;
11453	pDbgState->fModifiedXcptBitmap = false;
11454	pDbgState->fClearCr0Mask = false;
11455	pDbgState->fClearCr4Mask = false;
11456	pDbgState->fCpe1Extra = 0;
11457	pDbgState->fCpe1Unwanted = 0;
11458	pDbgState->fCpe2Extra = 0;
11459	pDbgState->bmXcptExtra = 0;
11460	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11461	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11462	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11463	}
11464
11465
11466	/**
11467	* Updates the VMSC fields with changes requested by @a pDbgState.
11468	*
11469	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11470	* immediately before executing guest code, i.e. when interrupts are disabled.
11471	* We don't check status codes here as we cannot easily assert or return in the
11472	* latter case.
11473	*
11474	* @param pVCpu The cross context virtual CPU structure.
11475	* @param pVmxTransient The VMX-transient structure.
11476	* @param pDbgState The debug state.
11477	*/
11478	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11479	{
11480	/*
11481	* Ensure desired flags in VMCS control fields are set.
11482	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11483	*
11484	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11485	* there should be no stale data in pCtx at this point.
11486	*/
11487	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11488	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11489	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11490	{
11491	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11492	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11493	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11494	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11495	pDbgState->fModifiedProcCtls = true;
11496	}
11497
11498	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11499	{
11500	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11501	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11502	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11503	pDbgState->fModifiedProcCtls2 = true;
11504	}
11505
11506	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11507	{
11508	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11509	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11510	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11511	pDbgState->fModifiedXcptBitmap = true;
11512	}
11513
11514	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11515	{
11516	pVmcsInfo->u64Cr0Mask = 0;
11517	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, 0);
11518	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11519	}
11520
11521	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11522	{
11523	pVmcsInfo->u64Cr4Mask = 0;
11524	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, 0);
11525	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11526	}
11527
11528	NOREF(pVCpu);
11529	}
11530
11531
11532	/**
11533	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11534	* re-entry next time around.
11535	*
11536	* @returns Strict VBox status code (i.e. informational status codes too).
11537	* @param pVCpu The cross context virtual CPU structure.
11538	* @param pVmxTransient The VMX-transient structure.
11539	* @param pDbgState The debug state.
11540	* @param rcStrict The return code from executing the guest using single
11541	* stepping.
11542	*/
11543	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11544	VBOXSTRICTRC rcStrict)
11545	{
11546	/*
11547	* Restore VM-exit control settings as we may not reenter this function the
11548	* next time around.
11549	*/
11550	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11551
11552	/* We reload the initial value, trigger what we can of recalculations the
11553	next time around. From the looks of things, that's all that's required atm. */
11554	if (pDbgState->fModifiedProcCtls)
11555	{
11556	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11557	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11558	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11559	AssertRC(rc2);
11560	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11561	}
11562
11563	/* We're currently the only ones messing with this one, so just restore the
11564	cached value and reload the field. */
11565	if ( pDbgState->fModifiedProcCtls2
11566	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11567	{
11568	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11569	AssertRC(rc2);
11570	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11571	}
11572
11573	/* If we've modified the exception bitmap, we restore it and trigger
11574	reloading and partial recalculation the next time around. */
11575	if (pDbgState->fModifiedXcptBitmap)
11576	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11577
11578	return rcStrict;
11579	}
11580
11581
11582	/**
11583	* Configures VM-exit controls for current DBGF and DTrace settings.
11584	*
11585	* This updates @a pDbgState and the VMCS execution control fields to reflect
11586	* the necessary VM-exits demanded by DBGF and DTrace.
11587	*
11588	* @param pVCpu The cross context virtual CPU structure.
11589	* @param pVmxTransient The VMX-transient structure. May update
11590	* fUpdatedTscOffsettingAndPreemptTimer.
11591	* @param pDbgState The debug state.
11592	*/
11593	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11594	{
11595	/*
11596	* Take down the dtrace serial number so we can spot changes.
11597	*/
11598	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11599	ASMCompilerBarrier();
11600
11601	/*
11602	* We'll rebuild most of the middle block of data members (holding the
11603	* current settings) as we go along here, so start by clearing it all.
11604	*/
11605	pDbgState->bmXcptExtra = 0;
11606	pDbgState->fCpe1Extra = 0;
11607	pDbgState->fCpe1Unwanted = 0;
11608	pDbgState->fCpe2Extra = 0;
11609	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11610	pDbgState->bmExitsToCheck[i] = 0;
11611
11612	/*
11613	* Software interrupts (INT XXh) - no idea how to trigger these...
11614	*/
11615	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11616	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11617	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11618	{
11619	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11620	}
11621
11622	/*
11623	* INT3 breakpoints - triggered by #BP exceptions.
11624	*/
11625	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11626	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11627
11628	/*
11629	* Exception bitmap and XCPT events+probes.
11630	*/
11631	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11632	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11633	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11634
11635	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11636	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11637	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11638	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11639	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11640	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11641	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11642	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11643	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11644	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11645	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11646	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11647	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11648	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11649	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11650	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11651	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11652	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11653
11654	if (pDbgState->bmXcptExtra)
11655	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11656
11657	/*
11658	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11659	*
11660	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11661	* So, when adding/changing/removing please don't forget to update it.
11662	*
11663	* Some of the macros are picking up local variables to save horizontal space,
11664	* (being able to see it in a table is the lesser evil here).
11665	*/
11666	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11667	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11668	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11669	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11670	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11671	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11672	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11673	} else do { } while (0)
11674	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11675	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11676	{ \
11677	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11678	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11679	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11680	} else do { } while (0)
11681	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11682	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11683	{ \
11684	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11685	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11686	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11687	} else do { } while (0)
11688	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11689	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11690	{ \
11691	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11692	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11693	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11694	} else do { } while (0)
11695
11696	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11697	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11698	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11699	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11700	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11701
11702	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11703	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11704	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11705	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11706	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11707	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11708	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11709	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11710	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11711	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11712	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11713	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11714	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11715	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11716	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11717	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11718	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11719	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11720	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11721	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11722	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11723	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11724	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11725	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11726	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11727	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11728	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11729	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11730	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11731	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11732	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11733	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11734	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11735	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11736	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11737	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11738
11739	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11740	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11741	{
11742	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11743	\| CPUMCTX_EXTRN_APIC_TPR);
11744	AssertRC(rc);
11745
11746	#if 0 /** @todo fix me */
11747	pDbgState->fClearCr0Mask = true;
11748	pDbgState->fClearCr4Mask = true;
11749	#endif
11750	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11751	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11752	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11753	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11754	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11755	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11756	require clearing here and in the loop if we start using it. */
11757	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11758	}
11759	else
11760	{
11761	if (pDbgState->fClearCr0Mask)
11762	{
11763	pDbgState->fClearCr0Mask = false;
11764	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11765	}
11766	if (pDbgState->fClearCr4Mask)
11767	{
11768	pDbgState->fClearCr4Mask = false;
11769	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11770	}
11771	}
11772	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11773	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11774
11775	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11776	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11777	{
11778	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11779	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11780	}
11781	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11782	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11783
11784	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11785	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11786	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11787	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11788	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11789	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11790	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11791	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11792	#if 0 /** @todo too slow, fix handler. */
11793	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11794	#endif
11795	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11796
11797	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11798	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11799	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11800	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11801	{
11802	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11803	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11804	}
11805	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11806	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11807	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11808	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11809
11810	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11811	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11812	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11813	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11814	{
11815	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11816	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11817	}
11818	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11819	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11820	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11821	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11822
11823	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11824	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11825	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11826	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11827	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11828	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11829	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11830	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11831	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11832	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11833	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11834	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11835	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11836	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11837	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11838	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11839	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11840	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11841	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11842	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11843	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11844	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11845
11846	#undef IS_EITHER_ENABLED
11847	#undef SET_ONLY_XBM_IF_EITHER_EN
11848	#undef SET_CPE1_XBM_IF_EITHER_EN
11849	#undef SET_CPEU_XBM_IF_EITHER_EN
11850	#undef SET_CPE2_XBM_IF_EITHER_EN
11851
11852	/*
11853	* Sanitize the control stuff.
11854	*/
11855	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11856	if (pDbgState->fCpe2Extra)
11857	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11858	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11859	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11860	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11861	{
11862	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11863	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11864	}
11865
11866	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11867	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11868	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11869	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11870	}
11871
11872
11873	/**
11874	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11875	* appropriate.
11876	*
11877	* The caller has checked the VM-exit against the
11878	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11879	* already, so we don't have to do that either.
11880	*
11881	* @returns Strict VBox status code (i.e. informational status codes too).
11882	* @param pVCpu The cross context virtual CPU structure.
11883	* @param pVmxTransient The VMX-transient structure.
11884	* @param uExitReason The VM-exit reason.
11885	*
11886	* @remarks The name of this function is displayed by dtrace, so keep it short
11887	* and to the point. No longer than 33 chars long, please.
11888	*/
11889	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11890	{
11891	/*
11892	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11893	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11894	*
11895	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11896	* does. Must add/change/remove both places. Same ordering, please.
11897	*
11898	* Added/removed events must also be reflected in the next section
11899	* where we dispatch dtrace events.
11900	*/
11901	bool fDtrace1 = false;
11902	bool fDtrace2 = false;
11903	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11904	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11905	uint32_t uEventArg = 0;
11906	#define SET_EXIT(a_EventSubName) \
11907	do { \
11908	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11909	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11910	} while (0)
11911	#define SET_BOTH(a_EventSubName) \
11912	do { \
11913	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11914	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11915	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11916	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11917	} while (0)
11918	switch (uExitReason)
11919	{
11920	case VMX_EXIT_MTF:
11921	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11922
11923	case VMX_EXIT_XCPT_OR_NMI:
11924	{
11925	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11926	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11927	{
11928	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11929	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11930	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11931	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11932	{
11933	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11934	{
11935	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11936	uEventArg = pVmxTransient->uExitIntErrorCode;
11937	}
11938	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11939	switch (enmEvent1)
11940	{
11941	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11942	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11943	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11944	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11945	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11946	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11947	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11948	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11949	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11950	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11951	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11952	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11953	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11954	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11955	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11956	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11957	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11958	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11959	default: break;
11960	}
11961	}
11962	else
11963	AssertFailed();
11964	break;
11965
11966	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11967	uEventArg = idxVector;
11968	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11969	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11970	break;
11971	}
11972	break;
11973	}
11974
11975	case VMX_EXIT_TRIPLE_FAULT:
11976	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11977	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11978	break;
11979	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11980	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11981	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11982	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11983	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11984
11985	/* Instruction specific VM-exits: */
11986	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11987	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11988	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11989	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11990	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11991	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11992	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11993	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11994	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11995	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11996	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11997	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11998	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11999	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
12000	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
12001	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
12002	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12003	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12004	case VMX_EXIT_MOV_CRX:
12005	hmR0VmxReadExitQualVmcs(pVmxTransient);
12006	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12007	SET_BOTH(CRX_READ);
12008	else
12009	SET_BOTH(CRX_WRITE);
12010	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12011	break;
12012	case VMX_EXIT_MOV_DRX:
12013	hmR0VmxReadExitQualVmcs(pVmxTransient);
12014	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12015	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12016	SET_BOTH(DRX_READ);
12017	else
12018	SET_BOTH(DRX_WRITE);
12019	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12020	break;
12021	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12022	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12023	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12024	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12025	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12026	case VMX_EXIT_GDTR_IDTR_ACCESS:
12027	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12028	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12029	{
12030	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12031	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12032	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12033	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12034	}
12035	break;
12036
12037	case VMX_EXIT_LDTR_TR_ACCESS:
12038	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12039	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12040	{
12041	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12042	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12043	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12044	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12045	}
12046	break;
12047
12048	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12049	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12050	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12051	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12052	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12053	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12054	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12055	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12056	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12057	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12058	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12059
12060	/* Events that aren't relevant at this point. */
12061	case VMX_EXIT_EXT_INT:
12062	case VMX_EXIT_INT_WINDOW:
12063	case VMX_EXIT_NMI_WINDOW:
12064	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12065	case VMX_EXIT_PREEMPT_TIMER:
12066	case VMX_EXIT_IO_INSTR:
12067	break;
12068
12069	/* Errors and unexpected events. */
12070	case VMX_EXIT_INIT_SIGNAL:
12071	case VMX_EXIT_SIPI:
12072	case VMX_EXIT_IO_SMI:
12073	case VMX_EXIT_SMI:
12074	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12075	case VMX_EXIT_ERR_MSR_LOAD:
12076	case VMX_EXIT_ERR_MACHINE_CHECK:
12077	case VMX_EXIT_PML_FULL:
12078	case VMX_EXIT_VIRTUALIZED_EOI:
12079	break;
12080
12081	default:
12082	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12083	break;
12084	}
12085	#undef SET_BOTH
12086	#undef SET_EXIT
12087
12088	/*
12089	* Dtrace tracepoints go first. We do them here at once so we don't
12090	* have to copy the guest state saving and stuff a few dozen times.
12091	* Down side is that we've got to repeat the switch, though this time
12092	* we use enmEvent since the probes are a subset of what DBGF does.
12093	*/
12094	if (fDtrace1 \|\| fDtrace2)
12095	{
12096	hmR0VmxReadExitQualVmcs(pVmxTransient);
12097	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12098	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12099	switch (enmEvent1)
12100	{
12101	/** @todo consider which extra parameters would be helpful for each probe. */
12102	case DBGFEVENT_END: break;
12103	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12104	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12105	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12106	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12107	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12108	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12109	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12110	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12111	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12112	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12113	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12114	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12115	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12116	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12117	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12118	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12119	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12120	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12121	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12122	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12123	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12124	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12125	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12126	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12127	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12128	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12129	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12130	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12131	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12132	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12133	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12134	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12135	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12136	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12137	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12138	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12139	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12140	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12141	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12142	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12143	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12144	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12145	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12146	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12147	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12148	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12149	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12150	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12151	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12152	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12153	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12154	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12155	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12156	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12157	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12158	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12159	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12160	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12161	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12162	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12163	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12164	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12165	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12166	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12167	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12168	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12169	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12170	}
12171	switch (enmEvent2)
12172	{
12173	/** @todo consider which extra parameters would be helpful for each probe. */
12174	case DBGFEVENT_END: break;
12175	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12176	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12177	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12178	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12179	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12180	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12181	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12182	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12183	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12184	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12185	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12186	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12187	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12188	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12189	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12190	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12191	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12192	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12193	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12194	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12195	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12196	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12197	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12198	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12199	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12200	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12201	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12202	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12203	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12204	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12205	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12206	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12207	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12208	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12209	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12210	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12211	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12212	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12213	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12214	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12215	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12216	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12217	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12218	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12219	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12220	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12221	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12222	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12223	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12224	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12225	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12226	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12227	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12228	}
12229	}
12230
12231	/*
12232	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12233	* the DBGF call will do a full check).
12234	*
12235	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12236	* Note! If we have to events, we prioritize the first, i.e. the instruction
12237	* one, in order to avoid event nesting.
12238	*/
12239	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12240	if ( enmEvent1 != DBGFEVENT_END
12241	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12242	{
12243	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12244	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12245	if (rcStrict != VINF_SUCCESS)
12246	return rcStrict;
12247	}
12248	else if ( enmEvent2 != DBGFEVENT_END
12249	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12250	{
12251	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12252	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12253	if (rcStrict != VINF_SUCCESS)
12254	return rcStrict;
12255	}
12256
12257	return VINF_SUCCESS;
12258	}
12259
12260
12261	/**
12262	* Single-stepping VM-exit filtering.
12263	*
12264	* This is preprocessing the VM-exits and deciding whether we've gotten far
12265	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12266	* handling is performed.
12267	*
12268	* @returns Strict VBox status code (i.e. informational status codes too).
12269	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12270	* @param pVmxTransient The VMX-transient structure.
12271	* @param pDbgState The debug state.
12272	*/
12273	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12274	{
12275	/*
12276	* Expensive (saves context) generic dtrace VM-exit probe.
12277	*/
12278	uint32_t const uExitReason = pVmxTransient->uExitReason;
12279	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12280	{ /* more likely */ }
12281	else
12282	{
12283	hmR0VmxReadExitQualVmcs(pVmxTransient);
12284	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12285	AssertRC(rc);
12286	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12287	}
12288
12289	/*
12290	* Check for host NMI, just to get that out of the way.
12291	*/
12292	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12293	{ /* normally likely */ }
12294	else
12295	{
12296	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12297	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12298	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12299	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12300	}
12301
12302	/*
12303	* Check for single stepping event if we're stepping.
12304	*/
12305	if (pVCpu->hm.s.fSingleInstruction)
12306	{
12307	switch (uExitReason)
12308	{
12309	case VMX_EXIT_MTF:
12310	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12311
12312	/* Various events: */
12313	case VMX_EXIT_XCPT_OR_NMI:
12314	case VMX_EXIT_EXT_INT:
12315	case VMX_EXIT_TRIPLE_FAULT:
12316	case VMX_EXIT_INT_WINDOW:
12317	case VMX_EXIT_NMI_WINDOW:
12318	case VMX_EXIT_TASK_SWITCH:
12319	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12320	case VMX_EXIT_APIC_ACCESS:
12321	case VMX_EXIT_EPT_VIOLATION:
12322	case VMX_EXIT_EPT_MISCONFIG:
12323	case VMX_EXIT_PREEMPT_TIMER:
12324
12325	/* Instruction specific VM-exits: */
12326	case VMX_EXIT_CPUID:
12327	case VMX_EXIT_GETSEC:
12328	case VMX_EXIT_HLT:
12329	case VMX_EXIT_INVD:
12330	case VMX_EXIT_INVLPG:
12331	case VMX_EXIT_RDPMC:
12332	case VMX_EXIT_RDTSC:
12333	case VMX_EXIT_RSM:
12334	case VMX_EXIT_VMCALL:
12335	case VMX_EXIT_VMCLEAR:
12336	case VMX_EXIT_VMLAUNCH:
12337	case VMX_EXIT_VMPTRLD:
12338	case VMX_EXIT_VMPTRST:
12339	case VMX_EXIT_VMREAD:
12340	case VMX_EXIT_VMRESUME:
12341	case VMX_EXIT_VMWRITE:
12342	case VMX_EXIT_VMXOFF:
12343	case VMX_EXIT_VMXON:
12344	case VMX_EXIT_MOV_CRX:
12345	case VMX_EXIT_MOV_DRX:
12346	case VMX_EXIT_IO_INSTR:
12347	case VMX_EXIT_RDMSR:
12348	case VMX_EXIT_WRMSR:
12349	case VMX_EXIT_MWAIT:
12350	case VMX_EXIT_MONITOR:
12351	case VMX_EXIT_PAUSE:
12352	case VMX_EXIT_GDTR_IDTR_ACCESS:
12353	case VMX_EXIT_LDTR_TR_ACCESS:
12354	case VMX_EXIT_INVEPT:
12355	case VMX_EXIT_RDTSCP:
12356	case VMX_EXIT_INVVPID:
12357	case VMX_EXIT_WBINVD:
12358	case VMX_EXIT_XSETBV:
12359	case VMX_EXIT_RDRAND:
12360	case VMX_EXIT_INVPCID:
12361	case VMX_EXIT_VMFUNC:
12362	case VMX_EXIT_RDSEED:
12363	case VMX_EXIT_XSAVES:
12364	case VMX_EXIT_XRSTORS:
12365	{
12366	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12367	AssertRCReturn(rc, rc);
12368	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12369	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12370	return VINF_EM_DBG_STEPPED;
12371	break;
12372	}
12373
12374	/* Errors and unexpected events: */
12375	case VMX_EXIT_INIT_SIGNAL:
12376	case VMX_EXIT_SIPI:
12377	case VMX_EXIT_IO_SMI:
12378	case VMX_EXIT_SMI:
12379	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12380	case VMX_EXIT_ERR_MSR_LOAD:
12381	case VMX_EXIT_ERR_MACHINE_CHECK:
12382	case VMX_EXIT_PML_FULL:
12383	case VMX_EXIT_VIRTUALIZED_EOI:
12384	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12385	break;
12386
12387	default:
12388	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12389	break;
12390	}
12391	}
12392
12393	/*
12394	* Check for debugger event breakpoints and dtrace probes.
12395	*/
12396	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12397	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12398	{
12399	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12400	if (rcStrict != VINF_SUCCESS)
12401	return rcStrict;
12402	}
12403
12404	/*
12405	* Normal processing.
12406	*/
12407	#ifdef HMVMX_USE_FUNCTION_TABLE
12408	return g_aVMExitHandlers[uExitReason].pfn(pVCpu, pVmxTransient);
12409	#else
12410	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12411	#endif
12412	}
12413
12414
12415	/**
12416	* Single steps guest code using hardware-assisted VMX.
12417	*
12418	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12419	* but single-stepping through the hypervisor debugger.
12420	*
12421	* @returns Strict VBox status code (i.e. informational status codes too).
12422	* @param pVCpu The cross context virtual CPU structure.
12423	* @param pcLoops Pointer to the number of executed loops.
12424	*
12425	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12426	*/
12427	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
12428	{
12429	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12430	Assert(pcLoops);
12431	Assert(*pcLoops <= cMaxResumeLoops);
12432
12433	VMXTRANSIENT VmxTransient;
12434	RT_ZERO(VmxTransient);
12435	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12436
12437	/* Set HMCPU indicators. */
12438	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12439	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12440	pVCpu->hm.s.fDebugWantRdTscExit = false;
12441	pVCpu->hm.s.fUsingDebugLoop = true;
12442
12443	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12444	VMXRUNDBGSTATE DbgState;
12445	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12446	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12447
12448	/*
12449	* The loop.
12450	*/
12451	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12452	for (;;)
12453	{
12454	Assert(!HMR0SuspendPending());
12455	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12456	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12457	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12458
12459	/* Set up VM-execution controls the next two can respond to. */
12460	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12461
12462	/*
12463	* Preparatory work for running guest code, this may force us to
12464	* return to ring-3.
12465	*
12466	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12467	*/
12468	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12469	if (rcStrict != VINF_SUCCESS)
12470	break;
12471
12472	/* Interrupts are disabled at this point! */
12473	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12474
12475	/* Override any obnoxious code in the above two calls. */
12476	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12477
12478	/*
12479	* Finally execute the guest.
12480	*/
12481	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12482
12483	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12484	/* Interrupts are re-enabled at this point! */
12485
12486	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12487	if (RT_SUCCESS(rcRun))
12488	{ /* very likely */ }
12489	else
12490	{
12491	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12492	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12493	return rcRun;
12494	}
12495
12496	/* Profile the VM-exit. */
12497	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12498	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12499	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12500	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12501	HMVMX_START_EXIT_DISPATCH_PROF();
12502
12503	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12504
12505	/*
12506	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12507	*/
12508	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12509	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12510	if (rcStrict != VINF_SUCCESS)
12511	break;
12512	if (++(*pcLoops) > cMaxResumeLoops)
12513	{
12514	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12515	rcStrict = VINF_EM_RAW_INTERRUPT;
12516	break;
12517	}
12518
12519	/*
12520	* Stepping: Did the RIP change, if so, consider it a single step.
12521	* Otherwise, make sure one of the TFs gets set.
12522	*/
12523	if (fStepping)
12524	{
12525	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12526	AssertRC(rc);
12527	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12528	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12529	{
12530	rcStrict = VINF_EM_DBG_STEPPED;
12531	break;
12532	}
12533	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12534	}
12535
12536	/*
12537	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12538	*/
12539	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12540	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12541
12542	/* Restore all controls applied by hmR0VmxPreRunGuestDebugStateApply above. */
12543	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12544	Assert(rcStrict == VINF_SUCCESS);
12545	}
12546
12547	/*
12548	* Clear the X86_EFL_TF if necessary.
12549	*/
12550	if (pVCpu->hm.s.fClearTrapFlag)
12551	{
12552	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12553	AssertRC(rc);
12554	pVCpu->hm.s.fClearTrapFlag = false;
12555	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12556	}
12557	/** @todo there seems to be issues with the resume flag when the monitor trap
12558	* flag is pending without being used. Seen early in bios init when
12559	* accessing APIC page in protected mode. */
12560
12561	/* Restore HMCPU indicators. */
12562	pVCpu->hm.s.fUsingDebugLoop = false;
12563	pVCpu->hm.s.fDebugWantRdTscExit = false;
12564	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12565
12566	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12567	return rcStrict;
12568	}
12569
12570
12571	/** @} */
12572
12573
12574	/**
12575	* Checks if any expensive dtrace probes are enabled and we should go to the
12576	* debug loop.
12577	*
12578	* @returns true if we should use debug loop, false if not.
12579	*/
12580	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12581	{
12582	/* It's probably faster to OR the raw 32-bit counter variables together.
12583	Since the variables are in an array and the probes are next to one
12584	another (more or less), we have good locality. So, better read
12585	eight-nine cache lines ever time and only have one conditional, than
12586	128+ conditionals, right? */
12587	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12588	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12589	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12590	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12591	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12592	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12593	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12594	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12595	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12596	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12597	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12598	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12599	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12600	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12601	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12602	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12603	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12604	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12605	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12606	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12607	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12608	) != 0
12609	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12610	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12611	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12612	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12613	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12614	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12615	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12616	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12617	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12618	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12619	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12620	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12621	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12622	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12623	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12624	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12625	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12626	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12627	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12628	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12629	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12630	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12631	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12632	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12633	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12634	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12635	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12636	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12637	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12638	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12639	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12640	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12641	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12642	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12643	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12644	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12645	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12646	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12647	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12648	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12649	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12650	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12651	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12652	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12653	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12654	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12655	) != 0
12656	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12657	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12658	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12659	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12660	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12661	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12662	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12663	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12664	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12665	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12666	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12667	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12668	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12669	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12670	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12671	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12672	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12673	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12674	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12675	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12676	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12677	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12678	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12679	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12680	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12681	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12682	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12683	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12684	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12685	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12686	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12687	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12688	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12689	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12690	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12691	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12692	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12693	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12694	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12695	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12696	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12697	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12698	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12699	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12700	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12701	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12702	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12703	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12704	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12705	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12706	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12707	) != 0;
12708	}
12709
12710
12711	/**
12712	* Runs the guest using hardware-assisted VMX.
12713	*
12714	* @returns Strict VBox status code (i.e. informational status codes too).
12715	* @param pVCpu The cross context virtual CPU structure.
12716	*/
12717	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu)
12718	{
12719	AssertPtr(pVCpu);
12720	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12721	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12722	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12723	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12724
12725	VBOXSTRICTRC rcStrict;
12726	uint32_t cLoops = 0;
12727	for (;;)
12728	{
12729	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12730	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12731	#else
12732	NOREF(pCtx);
12733	bool const fInNestedGuestMode = false;
12734	#endif
12735	if (!fInNestedGuestMode)
12736	{
12737	if ( !pVCpu->hm.s.fUseDebugLoop
12738	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12739	&& !DBGFIsStepping(pVCpu)
12740	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12741	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12742	else
12743	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12744	}
12745	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12746	else
12747	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12748
12749	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12750	{
12751	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12752	continue;
12753	}
12754	if (rcStrict == VINF_VMX_VMEXIT)
12755	{
12756	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12757	continue;
12758	}
12759	#endif
12760	break;
12761	}
12762
12763	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12764	switch (rcLoop)
12765	{
12766	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12767	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12768	}
12769
12770	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12771	if (RT_FAILURE(rc2))
12772	{
12773	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12774	rcStrict = rc2;
12775	}
12776	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12777	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12778	return rcStrict;
12779	}
12780
12781
12782	#ifndef HMVMX_USE_FUNCTION_TABLE
12783	/**
12784	* Handles a guest VM-exit from hardware-assisted VMX execution.
12785	*
12786	* @returns Strict VBox status code (i.e. informational status codes too).
12787	* @param pVCpu The cross context virtual CPU structure.
12788	* @param pVmxTransient The VMX-transient structure.
12789	*/
12790	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12791	{
12792	#ifdef DEBUG_ramshankar
12793	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12794	do { \
12795	if (a_fSave != 0) \
12796	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12797	VBOXSTRICTRC rcStrict = a_CallExpr; \
12798	if (a_fSave != 0) \
12799	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12800	return rcStrict; \
12801	} while (0)
12802	#else
12803	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12804	#endif
12805	uint32_t const uExitReason = pVmxTransient->uExitReason;
12806	switch (uExitReason)
12807	{
12808	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12809	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12810	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12811	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12812	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12813	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12814	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12815	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12816	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12817	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12818	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12819	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12820	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12821	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12822	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12823	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12824	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12825	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12826	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12827	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12828	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12829	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12830	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12831	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12832	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12833	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12834	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12835	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12836	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12837	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12838	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12839	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12840	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12841	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12842	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12843	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12844	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12845	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12846	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12847	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12848	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12849	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12850	#else
12851	case VMX_EXIT_VMCLEAR:
12852	case VMX_EXIT_VMLAUNCH:
12853	case VMX_EXIT_VMPTRLD:
12854	case VMX_EXIT_VMPTRST:
12855	case VMX_EXIT_VMREAD:
12856	case VMX_EXIT_VMRESUME:
12857	case VMX_EXIT_VMWRITE:
12858	case VMX_EXIT_VMXOFF:
12859	case VMX_EXIT_VMXON:
12860	case VMX_EXIT_INVVPID:
12861	case VMX_EXIT_INVEPT:
12862	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12863	#endif
12864
12865	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12866	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12867	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12868
12869	case VMX_EXIT_INIT_SIGNAL:
12870	case VMX_EXIT_SIPI:
12871	case VMX_EXIT_IO_SMI:
12872	case VMX_EXIT_SMI:
12873	case VMX_EXIT_ERR_MSR_LOAD:
12874	case VMX_EXIT_ERR_MACHINE_CHECK:
12875	case VMX_EXIT_PML_FULL:
12876	case VMX_EXIT_VIRTUALIZED_EOI:
12877	case VMX_EXIT_GDTR_IDTR_ACCESS:
12878	case VMX_EXIT_LDTR_TR_ACCESS:
12879	case VMX_EXIT_APIC_WRITE:
12880	case VMX_EXIT_RDRAND:
12881	case VMX_EXIT_RSM:
12882	case VMX_EXIT_VMFUNC:
12883	case VMX_EXIT_ENCLS:
12884	case VMX_EXIT_RDSEED:
12885	case VMX_EXIT_XSAVES:
12886	case VMX_EXIT_XRSTORS:
12887	case VMX_EXIT_UMWAIT:
12888	case VMX_EXIT_TPAUSE:
12889	default:
12890	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12891	}
12892	#undef VMEXIT_CALL_RET
12893	}
12894	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12895
12896
12897	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12898	/**
12899	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12900	*
12901	* @returns Strict VBox status code (i.e. informational status codes too).
12902	* @param pVCpu The cross context virtual CPU structure.
12903	* @param pVmxTransient The VMX-transient structure.
12904	*/
12905	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12906	{
12907	uint32_t const uExitReason = pVmxTransient->uExitReason;
12908	switch (uExitReason)
12909	{
12910	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12911	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12912	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12913	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12914	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12915
12916	/*
12917	* We shouldn't direct host physical interrupts to the nested-guest.
12918	*/
12919	case VMX_EXIT_EXT_INT:
12920	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12921
12922	/*
12923	* Instructions that cause VM-exits unconditionally or the condition is
12924	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12925	* happens, it's guaranteed to be a nested-guest VM-exit).
12926	*
12927	* - Provides VM-exit instruction length ONLY.
12928	*/
12929	case VMX_EXIT_CPUID: /* Unconditional. */
12930	case VMX_EXIT_VMCALL:
12931	case VMX_EXIT_GETSEC:
12932	case VMX_EXIT_INVD:
12933	case VMX_EXIT_XSETBV:
12934	case VMX_EXIT_VMLAUNCH:
12935	case VMX_EXIT_VMRESUME:
12936	case VMX_EXIT_VMXOFF:
12937	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
12938	case VMX_EXIT_VMFUNC:
12939	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12940
12941	/*
12942	* Instructions that cause VM-exits unconditionally or the condition is
12943	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12944	* happens, it's guaranteed to be a nested-guest VM-exit).
12945	*
12946	* - Provides VM-exit instruction length.
12947	* - Provides VM-exit information.
12948	* - Optionally provides Exit qualification.
12949	*
12950	* Since Exit qualification is 0 for all VM-exits where it is not
12951	* applicable, reading and passing it to the guest should produce
12952	* defined behavior.
12953	*
12954	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12955	*/
12956	case VMX_EXIT_INVEPT: /* Unconditional. */
12957	case VMX_EXIT_INVVPID:
12958	case VMX_EXIT_VMCLEAR:
12959	case VMX_EXIT_VMPTRLD:
12960	case VMX_EXIT_VMPTRST:
12961	case VMX_EXIT_VMXON:
12962	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
12963	case VMX_EXIT_LDTR_TR_ACCESS:
12964	case VMX_EXIT_RDRAND:
12965	case VMX_EXIT_RDSEED:
12966	case VMX_EXIT_XSAVES:
12967	case VMX_EXIT_XRSTORS:
12968	case VMX_EXIT_UMWAIT:
12969	case VMX_EXIT_TPAUSE:
12970	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12971
12972	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12973	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12974	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12975	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12976	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12977	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12978	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12979	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12980	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12981	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12982	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12983	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12984	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12985	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12986	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12987	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12988	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12989	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12990	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12991
12992	case VMX_EXIT_PREEMPT_TIMER:
12993	{
12994	/** @todo NSTVMX: Preempt timer. */
12995	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12996	}
12997
12998	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12999	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
13000
13001	case VMX_EXIT_VMREAD:
13002	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13003
13004	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13005	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13006
13007	case VMX_EXIT_INIT_SIGNAL:
13008	case VMX_EXIT_SIPI:
13009	case VMX_EXIT_IO_SMI:
13010	case VMX_EXIT_SMI:
13011	case VMX_EXIT_ERR_MSR_LOAD:
13012	case VMX_EXIT_ERR_MACHINE_CHECK:
13013	case VMX_EXIT_PML_FULL:
13014	case VMX_EXIT_RSM:
13015	default:
13016	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13017	}
13018	}
13019	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13020
13021
13022	/** @name VM-exit helpers.
13023	* @{
13024	*/
13025	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13026	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13027	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13028
13029	/** Macro for VM-exits called unexpectedly. */
13030	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
13031	do { \
13032	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
13033	return VERR_VMX_UNEXPECTED_EXIT; \
13034	} while (0)
13035
13036	#ifdef VBOX_STRICT
13037	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13038	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13039	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13040
13041	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13042	do { \
13043	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13044	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13045	} while (0)
13046
13047	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13048	do { \
13049	AssertPtr((a_pVCpu)); \
13050	AssertPtr((a_pVmxTransient)); \
13051	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13052	Assert((a_pVmxTransient)->pVmcsInfo); \
13053	Assert(ASMIntAreEnabled()); \
13054	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13055	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13056	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
13057	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13058	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13059	HMVMX_ASSERT_PREEMPT_CPUID(); \
13060	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13061	} while (0)
13062
13063	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13064	do { \
13065	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13066	Assert((a_pVmxTransient)->fIsNestedGuest); \
13067	} while (0)
13068
13069	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13070	do { \
13071	Log4Func(("\n")); \
13072	} while (0)
13073	#else
13074	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13075	do { \
13076	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13077	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13078	} while (0)
13079
13080	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13081	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13082
13083	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13084	#endif
13085
13086	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13087	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13088	* guests that attempted to execute a VMX instruction. */
13089	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13090	do \
13091	{ \
13092	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13093	if (rcStrictTmp == VINF_SUCCESS) \
13094	{ /* likely */ } \
13095	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13096	{ \
13097	Assert((a_pVCpu)->hm.s.Event.fPending); \
13098	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13099	return VINF_SUCCESS; \
13100	} \
13101	else \
13102	{ \
13103	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13104	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13105	} \
13106	} while (0)
13107
13108	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13109	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13110	do \
13111	{ \
13112	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13113	(a_pGCPtrEffAddr)); \
13114	if (rcStrictTmp == VINF_SUCCESS) \
13115	{ /* likely */ } \
13116	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13117	{ \
13118	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13119	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13120	NOREF(uXcptTmp); \
13121	return VINF_SUCCESS; \
13122	} \
13123	else \
13124	{ \
13125	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13126	return rcStrictTmp; \
13127	} \
13128	} while (0)
13129	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13130
13131
13132	/**
13133	* Advances the guest RIP by the specified number of bytes.
13134	*
13135	* @param pVCpu The cross context virtual CPU structure.
13136	* @param cbInstr Number of bytes to advance the RIP by.
13137	*
13138	* @remarks No-long-jump zone!!!
13139	*/
13140	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
13141	{
13142	/* Advance the RIP. */
13143	pVCpu->cpum.GstCtx.rip += cbInstr;
13144	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13145
13146	/* Update interrupt inhibition. */
13147	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13148	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13149	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13150	}
13151
13152
13153	/**
13154	* Advances the guest RIP after reading it from the VMCS.
13155	*
13156	* @returns VBox status code, no informational status codes.
13157	* @param pVCpu The cross context virtual CPU structure.
13158	* @param pVmxTransient The VMX-transient structure.
13159	*
13160	* @remarks No-long-jump zone!!!
13161	*/
13162	static int hmR0VmxAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13163	{
13164	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13165	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13166	AssertRCReturn(rc, rc);
13167
13168	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
13169	return VINF_SUCCESS;
13170	}
13171
13172
13173	/**
13174	* Handle a condition that occurred while delivering an event through the guest or
13175	* nested-guest IDT.
13176	*
13177	* @returns Strict VBox status code (i.e. informational status codes too).
13178	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13179	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13180	* to continue execution of the guest which will delivery the \#DF.
13181	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13182	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13183	*
13184	* @param pVCpu The cross context virtual CPU structure.
13185	* @param pVmxTransient The VMX-transient structure.
13186	*
13187	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13188	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
13189	* is due to an EPT violation, PML full or SPP-related event.
13190	*
13191	* @remarks No-long-jump zone!!!
13192	*/
13193	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13194	{
13195	Assert(!pVCpu->hm.s.Event.fPending);
13196	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
13197	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13198	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13199	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13200	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
13201
13202	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13203	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13204	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
13205	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13206	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
13207	{
13208	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
13209	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
13210
13211	/*
13212	* If the event was a software interrupt (generated with INT n) or a software exception
13213	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13214	* can handle the VM-exit and continue guest execution which will re-execute the
13215	* instruction rather than re-injecting the exception, as that can cause premature
13216	* trips to ring-3 before injection and involve TRPM which currently has no way of
13217	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13218	* the problem).
13219	*/
13220	IEMXCPTRAISE enmRaise;
13221	IEMXCPTRAISEINFO fRaiseInfo;
13222	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13223	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13224	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13225	{
13226	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13227	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13228	}
13229	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
13230	{
13231	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13232	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13233	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
13234
13235	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13236	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13237
13238	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13239
13240	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13241	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13242	{
13243	pVmxTransient->fVectoringPF = true;
13244	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13245	}
13246	}
13247	else
13248	{
13249	/*
13250	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13251	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13252	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13253	*/
13254	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13255	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13256	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13257	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13258	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13259	}
13260
13261	/*
13262	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13263	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13264	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13265	* subsequent VM-entry would fail, see @bugref{7445}.
13266	*
13267	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13268	*/
13269	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13270	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
13271	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13272	&& CPUMIsGuestNmiBlocking(pVCpu))
13273	{
13274	CPUMSetGuestNmiBlocking(pVCpu, false);
13275	}
13276
13277	switch (enmRaise)
13278	{
13279	case IEMXCPTRAISE_CURRENT_XCPT:
13280	{
13281	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
13282	Assert(rcStrict == VINF_SUCCESS);
13283	break;
13284	}
13285
13286	case IEMXCPTRAISE_PREV_EVENT:
13287	{
13288	uint32_t u32ErrCode;
13289	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13290	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13291	else
13292	u32ErrCode = 0;
13293
13294	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13295	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13296	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13297	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13298
13299	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13300	pVCpu->hm.s.Event.u32ErrCode));
13301	Assert(rcStrict == VINF_SUCCESS);
13302	break;
13303	}
13304
13305	case IEMXCPTRAISE_REEXEC_INSTR:
13306	Assert(rcStrict == VINF_SUCCESS);
13307	break;
13308
13309	case IEMXCPTRAISE_DOUBLE_FAULT:
13310	{
13311	/*
13312	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13313	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13314	*/
13315	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13316	{
13317	pVmxTransient->fVectoringDoublePF = true;
13318	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13319	pVCpu->cpum.GstCtx.cr2));
13320	rcStrict = VINF_SUCCESS;
13321	}
13322	else
13323	{
13324	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13325	hmR0VmxSetPendingXcptDF(pVCpu);
13326	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13327	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13328	rcStrict = VINF_HM_DOUBLE_FAULT;
13329	}
13330	break;
13331	}
13332
13333	case IEMXCPTRAISE_TRIPLE_FAULT:
13334	{
13335	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13336	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13337	rcStrict = VINF_EM_RESET;
13338	break;
13339	}
13340
13341	case IEMXCPTRAISE_CPU_HANG:
13342	{
13343	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13344	rcStrict = VERR_EM_GUEST_CPU_HANG;
13345	break;
13346	}
13347
13348	default:
13349	{
13350	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13351	rcStrict = VERR_VMX_IPE_2;
13352	break;
13353	}
13354	}
13355	}
13356	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13357	&& !CPUMIsGuestNmiBlocking(pVCpu))
13358	{
13359	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13360	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13361	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13362	{
13363	/*
13364	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13365	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13366	* that virtual NMIs remain blocked until the IRET execution is completed.
13367	*
13368	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13369	*/
13370	CPUMSetGuestNmiBlocking(pVCpu, true);
13371	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13372	}
13373	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13374	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13375	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13376	{
13377	/*
13378	* Execution of IRET caused an EPT violation, page-modification log-full event or
13379	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13380	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13381	* that virtual NMIs remain blocked until the IRET execution is completed.
13382	*
13383	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13384	*/
13385	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13386	{
13387	CPUMSetGuestNmiBlocking(pVCpu, true);
13388	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13389	}
13390	}
13391	}
13392
13393	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13394	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13395	return rcStrict;
13396	}
13397
13398
13399	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13400	/**
13401	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13402	* guest attempting to execute a VMX instruction.
13403	*
13404	* @returns Strict VBox status code (i.e. informational status codes too).
13405	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13406	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13407	*
13408	* @param pVCpu The cross context virtual CPU structure.
13409	* @param uExitReason The VM-exit reason.
13410	*
13411	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13412	* @remarks No-long-jump zone!!!
13413	*/
13414	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
13415	{
13416	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13417	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13418
13419	/*
13420	* The physical CPU would have already checked the CPU mode/code segment.
13421	* We shall just assert here for paranoia.
13422	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13423	*/
13424	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13425	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13426	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13427
13428	if (uExitReason == VMX_EXIT_VMXON)
13429	{
13430	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13431
13432	/*
13433	* We check CR4.VMXE because it is required to be always set while in VMX operation
13434	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13435	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13436	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13437	*/
13438	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13439	{
13440	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13441	hmR0VmxSetPendingXcptUD(pVCpu);
13442	return VINF_HM_PENDING_XCPT;
13443	}
13444	}
13445	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13446	{
13447	/*
13448	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13449	* (other than VMXON), we need to raise a #UD.
13450	*/
13451	Log4Func(("Not in VMX root mode -> #UD\n"));
13452	hmR0VmxSetPendingXcptUD(pVCpu);
13453	return VINF_HM_PENDING_XCPT;
13454	}
13455
13456	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13457	return VINF_SUCCESS;
13458	}
13459
13460
13461	/**
13462	* Decodes the memory operand of an instruction that caused a VM-exit.
13463	*
13464	* The Exit qualification field provides the displacement field for memory
13465	* operand instructions, if any.
13466	*
13467	* @returns Strict VBox status code (i.e. informational status codes too).
13468	* @retval VINF_SUCCESS if the operand was successfully decoded.
13469	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13470	* operand.
13471	* @param pVCpu The cross context virtual CPU structure.
13472	* @param uExitInstrInfo The VM-exit instruction information field.
13473	* @param enmMemAccess The memory operand's access type (read or write).
13474	* @param GCPtrDisp The instruction displacement field, if any. For
13475	* RIP-relative addressing pass RIP + displacement here.
13476	* @param pGCPtrMem Where to store the effective destination memory address.
13477	*
13478	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13479	* virtual-8086 mode hence skips those checks while verifying if the
13480	* segment is valid.
13481	*/
13482	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13483	PRTGCPTR pGCPtrMem)
13484	{
13485	Assert(pGCPtrMem);
13486	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13487	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13488	\| CPUMCTX_EXTRN_CR0);
13489
13490	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13491	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13492	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13493
13494	VMXEXITINSTRINFO ExitInstrInfo;
13495	ExitInstrInfo.u = uExitInstrInfo;
13496	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13497	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13498	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13499	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13500	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13501	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13502	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13503	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13504	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13505
13506	/*
13507	* Validate instruction information.
13508	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13509	*/
13510	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13511	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13512	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13513	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13514	AssertLogRelMsgReturn(fIsMemOperand,
13515	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13516
13517	/*
13518	* Compute the complete effective address.
13519	*
13520	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13521	* See AMD spec. 4.5.2 "Segment Registers".
13522	*/
13523	RTGCPTR GCPtrMem = GCPtrDisp;
13524	if (fBaseRegValid)
13525	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13526	if (fIdxRegValid)
13527	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13528
13529	RTGCPTR const GCPtrOff = GCPtrMem;
13530	if ( !fIsLongMode
13531	\|\| iSegReg >= X86_SREG_FS)
13532	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13533	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13534
13535	/*
13536	* Validate effective address.
13537	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13538	*/
13539	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13540	Assert(cbAccess > 0);
13541	if (fIsLongMode)
13542	{
13543	if (X86_IS_CANONICAL(GCPtrMem))
13544	{
13545	*pGCPtrMem = GCPtrMem;
13546	return VINF_SUCCESS;
13547	}
13548
13549	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13550	* "Data Limit Checks in 64-bit Mode". */
13551	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13552	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13553	return VINF_HM_PENDING_XCPT;
13554	}
13555
13556	/*
13557	* This is a watered down version of iemMemApplySegment().
13558	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13559	* and segment CPL/DPL checks are skipped.
13560	*/
13561	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13562	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13563	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13564
13565	/* Check if the segment is present and usable. */
13566	if ( pSel->Attr.n.u1Present
13567	&& !pSel->Attr.n.u1Unusable)
13568	{
13569	Assert(pSel->Attr.n.u1DescType);
13570	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13571	{
13572	/* Check permissions for the data segment. */
13573	if ( enmMemAccess == VMXMEMACCESS_WRITE
13574	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13575	{
13576	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13577	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13578	return VINF_HM_PENDING_XCPT;
13579	}
13580
13581	/* Check limits if it's a normal data segment. */
13582	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13583	{
13584	if ( GCPtrFirst32 > pSel->u32Limit
13585	\|\| GCPtrLast32 > pSel->u32Limit)
13586	{
13587	Log4Func(("Data segment limit exceeded. "
13588	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13589	GCPtrLast32, pSel->u32Limit));
13590	if (iSegReg == X86_SREG_SS)
13591	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13592	else
13593	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13594	return VINF_HM_PENDING_XCPT;
13595	}
13596	}
13597	else
13598	{
13599	/* Check limits if it's an expand-down data segment.
13600	Note! The upper boundary is defined by the B bit, not the G bit! */
13601	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13602	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13603	{
13604	Log4Func(("Expand-down data segment limit exceeded. "
13605	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13606	GCPtrLast32, pSel->u32Limit));
13607	if (iSegReg == X86_SREG_SS)
13608	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13609	else
13610	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13611	return VINF_HM_PENDING_XCPT;
13612	}
13613	}
13614	}
13615	else
13616	{
13617	/* Check permissions for the code segment. */
13618	if ( enmMemAccess == VMXMEMACCESS_WRITE
13619	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13620	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13621	{
13622	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13623	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13624	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13625	return VINF_HM_PENDING_XCPT;
13626	}
13627
13628	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13629	if ( GCPtrFirst32 > pSel->u32Limit
13630	\|\| GCPtrLast32 > pSel->u32Limit)
13631	{
13632	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13633	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13634	if (iSegReg == X86_SREG_SS)
13635	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13636	else
13637	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13638	return VINF_HM_PENDING_XCPT;
13639	}
13640	}
13641	}
13642	else
13643	{
13644	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13645	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13646	return VINF_HM_PENDING_XCPT;
13647	}
13648
13649	*pGCPtrMem = GCPtrMem;
13650	return VINF_SUCCESS;
13651	}
13652	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13653
13654
13655	/**
13656	* VM-exit helper for LMSW.
13657	*/
13658	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13659	{
13660	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13661	AssertRCReturn(rc, rc);
13662
13663	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13664	AssertMsg( rcStrict == VINF_SUCCESS
13665	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13666
13667	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13668	if (rcStrict == VINF_IEM_RAISED_XCPT)
13669	{
13670	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13671	rcStrict = VINF_SUCCESS;
13672	}
13673
13674	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13675	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13676	return rcStrict;
13677	}
13678
13679
13680	/**
13681	* VM-exit helper for CLTS.
13682	*/
13683	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13684	{
13685	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13686	AssertRCReturn(rc, rc);
13687
13688	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13689	AssertMsg( rcStrict == VINF_SUCCESS
13690	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13691
13692	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13693	if (rcStrict == VINF_IEM_RAISED_XCPT)
13694	{
13695	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13696	rcStrict = VINF_SUCCESS;
13697	}
13698
13699	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13700	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13701	return rcStrict;
13702	}
13703
13704
13705	/**
13706	* VM-exit helper for MOV from CRx (CRx read).
13707	*/
13708	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13709	{
13710	Assert(iCrReg < 16);
13711	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13712
13713	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13714	AssertRCReturn(rc, rc);
13715
13716	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13717	AssertMsg( rcStrict == VINF_SUCCESS
13718	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13719
13720	if (iGReg == X86_GREG_xSP)
13721	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13722	else
13723	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13724	#ifdef VBOX_WITH_STATISTICS
13725	switch (iCrReg)
13726	{
13727	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13728	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13729	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13730	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13731	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13732	}
13733	#endif
13734	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13735	return rcStrict;
13736	}
13737
13738
13739	/**
13740	* VM-exit helper for MOV to CRx (CRx write).
13741	*/
13742	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13743	{
13744	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13745	AssertRCReturn(rc, rc);
13746
13747	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13748	AssertMsg( rcStrict == VINF_SUCCESS
13749	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13750	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13751
13752	switch (iCrReg)
13753	{
13754	case 0:
13755	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
13756	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13757	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13758	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13759	break;
13760
13761	case 2:
13762	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13763	/* Nothing to do here, CR2 it's not part of the VMCS. */
13764	break;
13765
13766	case 3:
13767	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13768	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13769	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13770	break;
13771
13772	case 4:
13773	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13774	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13775	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13776	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13777	break;
13778
13779	case 8:
13780	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13781	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13782	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13783	break;
13784
13785	default:
13786	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13787	break;
13788	}
13789
13790	if (rcStrict == VINF_IEM_RAISED_XCPT)
13791	{
13792	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13793	rcStrict = VINF_SUCCESS;
13794	}
13795	return rcStrict;
13796	}
13797
13798
13799	/**
13800	* VM-exit exception handler for \#PF (Page-fault exception).
13801	*
13802	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13803	*/
13804	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13805	{
13806	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13807	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13808	hmR0VmxReadExitQualVmcs(pVmxTransient);
13809
13810	if (!pVM->hm.s.fNestedPaging)
13811	{ /* likely */ }
13812	else
13813	{
13814	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13815	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13816	#endif
13817	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13818	if (!pVmxTransient->fVectoringDoublePF)
13819	{
13820	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13821	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13822	}
13823	else
13824	{
13825	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13826	Assert(!pVmxTransient->fIsNestedGuest);
13827	hmR0VmxSetPendingXcptDF(pVCpu);
13828	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13829	}
13830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13831	return VINF_SUCCESS;
13832	}
13833
13834	Assert(!pVmxTransient->fIsNestedGuest);
13835
13836	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13837	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13838	if (pVmxTransient->fVectoringPF)
13839	{
13840	Assert(pVCpu->hm.s.Event.fPending);
13841	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13842	}
13843
13844	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13845	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13846	AssertRCReturn(rc, rc);
13847
13848	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
13849	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
13850
13851	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13852	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13853
13854	Log4Func(("#PF: rc=%Rrc\n", rc));
13855	if (rc == VINF_SUCCESS)
13856	{
13857	/*
13858	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13859	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13860	*/
13861	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13862	TRPMResetTrap(pVCpu);
13863	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13864	return rc;
13865	}
13866
13867	if (rc == VINF_EM_RAW_GUEST_TRAP)
13868	{
13869	if (!pVmxTransient->fVectoringDoublePF)
13870	{
13871	/* It's a guest page fault and needs to be reflected to the guest. */
13872	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
13873	TRPMResetTrap(pVCpu);
13874	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13875	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13876	uGstErrorCode, pVmxTransient->uExitQual);
13877	}
13878	else
13879	{
13880	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13881	TRPMResetTrap(pVCpu);
13882	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13883	hmR0VmxSetPendingXcptDF(pVCpu);
13884	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13885	}
13886
13887	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13888	return VINF_SUCCESS;
13889	}
13890
13891	TRPMResetTrap(pVCpu);
13892	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13893	return rc;
13894	}
13895
13896
13897	/**
13898	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13899	*
13900	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13901	*/
13902	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13903	{
13904	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13905	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13906
13907	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13908	AssertRCReturn(rc, rc);
13909
13910	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13911	{
13912	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13913	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13914
13915	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13916	* provides VM-exit instruction length. If this causes problem later,
13917	* disassemble the instruction like it's done on AMD-V. */
13918	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13919	AssertRCReturn(rc2, rc2);
13920	return rc;
13921	}
13922
13923	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
13924	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13925	return VINF_SUCCESS;
13926	}
13927
13928
13929	/**
13930	* VM-exit exception handler for \#BP (Breakpoint exception).
13931	*
13932	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13933	*/
13934	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13935	{
13936	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13937	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13938
13939	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13940	AssertRCReturn(rc, rc);
13941
13942	if (!pVmxTransient->fIsNestedGuest)
13943	rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13944	else
13945	rc = VINF_EM_RAW_GUEST_TRAP;
13946
13947	if (rc == VINF_EM_RAW_GUEST_TRAP)
13948	{
13949	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13950	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13951	rc = VINF_SUCCESS;
13952	}
13953
13954	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
13955	return rc;
13956	}
13957
13958
13959	/**
13960	* VM-exit exception handler for \#AC (Alignment-check exception).
13961	*
13962	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13963	*/
13964	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13965	{
13966	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13967	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13968
13969	/* Re-inject it. We'll detect any nesting before getting here. */
13970	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13971	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13972	return VINF_SUCCESS;
13973	}
13974
13975
13976	/**
13977	* VM-exit exception handler for \#DB (Debug exception).
13978	*
13979	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13980	*/
13981	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13982	{
13983	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13984	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13985
13986	/*
13987	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13988	*/
13989	hmR0VmxReadExitQualVmcs(pVmxTransient);
13990
13991	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13992	uint64_t const uDR6 = X86_DR6_INIT_VAL
13993	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13994	\| X86_DR6_BD \| X86_DR6_BS));
13995
13996	int rc;
13997	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13998	if (!pVmxTransient->fIsNestedGuest)
13999	{
14000	rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14001
14002	/*
14003	* Prevents stepping twice over the same instruction when the guest is stepping using
14004	* EFLAGS.TF and the hypervisor debugger is stepping using MTF.
14005	* Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
14006	*/
14007	if ( rc == VINF_EM_DBG_STEPPED
14008	&& (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
14009	{
14010	Assert(pVCpu->hm.s.fSingleInstruction);
14011	rc = VINF_EM_RAW_GUEST_TRAP;
14012	}
14013	}
14014	else
14015	rc = VINF_EM_RAW_GUEST_TRAP;
14016	Log6Func(("rc=%Rrc\n", rc));
14017	if (rc == VINF_EM_RAW_GUEST_TRAP)
14018	{
14019	/*
14020	* The exception was for the guest. Update DR6, DR7.GD and
14021	* IA32_DEBUGCTL.LBR before forwarding it.
14022	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14023	*/
14024	VMMRZCallRing3Disable(pVCpu);
14025	HM_DISABLE_PREEMPT(pVCpu);
14026
14027	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14028	pCtx->dr[6] \|= uDR6;
14029	if (CPUMIsGuestDebugStateActive(pVCpu))
14030	ASMSetDR6(pCtx->dr[6]);
14031
14032	HM_RESTORE_PREEMPT();
14033	VMMRZCallRing3Enable(pVCpu);
14034
14035	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14036	AssertRCReturn(rc, rc);
14037
14038	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14039	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
14040
14041	/* Paranoia. */
14042	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
14043	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14044
14045	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
14046	AssertRC(rc);
14047
14048	/*
14049	* Raise #DB in the guest.
14050	*
14051	* It is important to reflect exactly what the VM-exit gave us (preserving the
14052	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14053	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14054	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14055	*
14056	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14057	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14058	*/
14059	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14060	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14061	return VINF_SUCCESS;
14062	}
14063
14064	/*
14065	* Not a guest trap, must be a hypervisor related debug event then.
14066	* Update DR6 in case someone is interested in it.
14067	*/
14068	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14069	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14070	CPUMSetHyperDR6(pVCpu, uDR6);
14071
14072	return rc;
14073	}
14074
14075
14076	/**
14077	* Hacks its way around the lovely mesa driver's backdoor accesses.
14078	*
14079	* @sa hmR0SvmHandleMesaDrvGp.
14080	*/
14081	static int hmR0VmxHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14082	{
14083	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14084	RT_NOREF(pCtx);
14085
14086	/* For now we'll just skip the instruction. */
14087	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14088	}
14089
14090
14091	/**
14092	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14093	* backdoor logging w/o checking what it is running inside.
14094	*
14095	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14096	* backdoor port and magic numbers loaded in registers.
14097	*
14098	* @returns true if it is, false if it isn't.
14099	* @sa hmR0SvmIsMesaDrvGp.
14100	*/
14101	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14102	{
14103	/* 0xed: IN eAX,dx */
14104	uint8_t abInstr[1];
14105	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
14106	return false;
14107
14108	/* Check that it is #GP(0). */
14109	if (pVmxTransient->uExitIntErrorCode != 0)
14110	return false;
14111
14112	/* Check magic and port. */
14113	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14114	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14115	if (pCtx->rax != UINT32_C(0x564d5868))
14116	return false;
14117	if (pCtx->dx != UINT32_C(0x5658))
14118	return false;
14119
14120	/* Flat ring-3 CS. */
14121	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14122	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14123	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14124	if (pCtx->cs.Attr.n.u2Dpl != 3)
14125	return false;
14126	if (pCtx->cs.u64Base != 0)
14127	return false;
14128
14129	/* Check opcode. */
14130	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14131	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14132	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14133	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14134	if (RT_FAILURE(rc))
14135	return false;
14136	if (abInstr[0] != 0xed)
14137	return false;
14138
14139	return true;
14140	}
14141
14142
14143	/**
14144	* VM-exit exception handler for \#GP (General-protection exception).
14145	*
14146	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
14147	*/
14148	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14149	{
14150	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14151	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14152
14153	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14154	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14155	if (pVmcsInfo->RealMode.fRealOnV86Active)
14156	{ /* likely */ }
14157	else
14158	{
14159	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14160	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14161	#endif
14162	/*
14163	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
14164	* executing a nested-guest, reflect #GP to the guest or nested-guest.
14165	*/
14166	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14167	AssertRCReturn(rc, rc);
14168	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14169	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14170
14171	if ( pVmxTransient->fIsNestedGuest
14172	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14173	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14174	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14175	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14176	else
14177	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14178	return rc;
14179	}
14180
14181	Assert(CPUMIsGuestInRealModeEx(pCtx));
14182	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14183	Assert(!pVmxTransient->fIsNestedGuest);
14184
14185	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14186	AssertRCReturn(rc, rc);
14187
14188	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14189	if (rcStrict == VINF_SUCCESS)
14190	{
14191	if (!CPUMIsGuestInRealModeEx(pCtx))
14192	{
14193	/*
14194	* The guest is no longer in real-mode, check if we can continue executing the
14195	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14196	*/
14197	pVmcsInfo->RealMode.fRealOnV86Active = false;
14198	if (HMCanExecuteVmxGuest(pVCpu->pVMR0, pVCpu, pCtx))
14199	{
14200	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14201	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14202	}
14203	else
14204	{
14205	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14206	rcStrict = VINF_EM_RESCHEDULE;
14207	}
14208	}
14209	else
14210	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14211	}
14212	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14213	{
14214	rcStrict = VINF_SUCCESS;
14215	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14216	}
14217	return VBOXSTRICTRC_VAL(rcStrict);
14218	}
14219
14220
14221	/**
14222	* VM-exit exception handler wrapper for all other exceptions that are not handled
14223	* by a specific handler.
14224	*
14225	* This simply re-injects the exception back into the VM without any special
14226	* processing.
14227	*
14228	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
14229	*/
14230	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14231	{
14232	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14233
14234	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14235	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14236	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14237	("uVector=%#x u32XcptBitmap=%#X32\n",
14238	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14239	NOREF(pVmcsInfo);
14240	#endif
14241
14242	/*
14243	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14244	* would have been handled while checking exits due to event delivery.
14245	*/
14246	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14247
14248	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14249	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14250	AssertRCReturn(rc, rc);
14251	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14252	#endif
14253
14254	#ifdef VBOX_WITH_STATISTICS
14255	switch (uVector)
14256	{
14257	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14258	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14259	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14260	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14261	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14262	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14263	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14264	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14265	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14266	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14267	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14268	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14269	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14270	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14271	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14272	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14273	default:
14274	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14275	break;
14276	}
14277	#endif
14278
14279	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14280	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
14281	NOREF(uVector);
14282
14283	/* Re-inject the original exception into the guest. */
14284	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14285	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14286	return VINF_SUCCESS;
14287	}
14288
14289
14290	/**
14291	* VM-exit exception handler for all exceptions (except NMIs!).
14292	*
14293	* @remarks This may be called for both guests and nested-guests. Take care to not
14294	* make assumptions and avoid doing anything that is not relevant when
14295	* executing a nested-guest (e.g., Mesa driver hacks).
14296	*/
14297	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14298	{
14299	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
14300
14301	/*
14302	* If this VM-exit occurred while delivering an event through the guest IDT, take
14303	* action based on the return code and additional hints (e.g. for page-faults)
14304	* that will be updated in the VMX transient structure.
14305	*/
14306	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14307	if (rcStrict == VINF_SUCCESS)
14308	{
14309	/*
14310	* If an exception caused a VM-exit due to delivery of an event, the original
14311	* event may have to be re-injected into the guest. We shall reinject it and
14312	* continue guest execution. However, page-fault is a complicated case and
14313	* needs additional processing done in hmR0VmxExitXcptPF().
14314	*/
14315	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14316	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14317	if ( !pVCpu->hm.s.Event.fPending
14318	\|\| uVector == X86_XCPT_PF)
14319	{
14320	switch (uVector)
14321	{
14322	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
14323	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
14324	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
14325	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
14326	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
14327	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
14328	default:
14329	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
14330	}
14331	}
14332	/* else: inject pending event before resuming guest execution. */
14333	}
14334	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
14335	{
14336	Assert(pVCpu->hm.s.Event.fPending);
14337	rcStrict = VINF_SUCCESS;
14338	}
14339
14340	return rcStrict;
14341	}
14342	/** @} */
14343
14344
14345	/** @name VM-exit handlers.
14346	* @{
14347	*/
14348	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14349	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14350	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14351
14352	/**
14353	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14354	*/
14355	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14356	{
14357	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14358	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14359	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14360	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14361	return VINF_SUCCESS;
14362	return VINF_EM_RAW_INTERRUPT;
14363	}
14364
14365
14366	/**
14367	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14368	* VM-exit.
14369	*/
14370	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14371	{
14372	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14373	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14374
14375	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14376
14377	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14378	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14379	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14380
14381	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14382	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14383	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14384	NOREF(pVmcsInfo);
14385
14386	VBOXSTRICTRC rcStrict;
14387	switch (uExitIntType)
14388	{
14389	/*
14390	* Host physical NMIs:
14391	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14392	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14393	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14394	*
14395	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14396	* See Intel spec. 27.5.5 "Updating Non-Register State".
14397	*/
14398	case VMX_EXIT_INT_INFO_TYPE_NMI:
14399	{
14400	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14401	break;
14402	}
14403
14404	/*
14405	* Privileged software exceptions (#DB from ICEBP),
14406	* Software exceptions (#BP and #OF),
14407	* Hardware exceptions:
14408	* Process the required exceptions and resume guest execution if possible.
14409	*/
14410	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14411	Assert(uVector == X86_XCPT_DB);
14412	RT_FALL_THRU();
14413	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14414	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14415	RT_FALL_THRU();
14416	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14417	{
14418	NOREF(uVector);
14419	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14420	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14421	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14422	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14423
14424	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14425	break;
14426	}
14427
14428	default:
14429	{
14430	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14431	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14432	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14433	break;
14434	}
14435	}
14436
14437	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14438	return rcStrict;
14439	}
14440
14441
14442	/**
14443	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14444	*/
14445	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14446	{
14447	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14448
14449	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14450	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14451	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14452
14453	/* Evaluate and deliver pending events and resume guest execution. */
14454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14455	return VINF_SUCCESS;
14456	}
14457
14458
14459	/**
14460	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14461	*/
14462	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14463	{
14464	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14465
14466	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14467	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14468	{
14469	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14470	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14471	}
14472
14473	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14474
14475	/*
14476	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14477	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14478	*/
14479	uint32_t fIntrState;
14480	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14481	AssertRC(rc);
14482	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14483	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14484	{
14485	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14486	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14487
14488	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14489	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14490	AssertRC(rc);
14491	}
14492
14493	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14494	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14495
14496	/* Evaluate and deliver pending events and resume guest execution. */
14497	return VINF_SUCCESS;
14498	}
14499
14500
14501	/**
14502	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14503	*/
14504	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14505	{
14506	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14507	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14508	}
14509
14510
14511	/**
14512	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14513	*/
14514	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14515	{
14516	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14517	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14518	}
14519
14520
14521	/**
14522	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14523	*/
14524	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14525	{
14526	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14527
14528	/*
14529	* Get the state we need and update the exit history entry.
14530	*/
14531	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14532	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14533
14534	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14535	AssertRCReturn(rc, rc);
14536
14537	VBOXSTRICTRC rcStrict;
14538	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14539	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14540	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14541	if (!pExitRec)
14542	{
14543	/*
14544	* Regular CPUID instruction execution.
14545	*/
14546	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
14547	if (rcStrict == VINF_SUCCESS)
14548	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14549	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14550	{
14551	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14552	rcStrict = VINF_SUCCESS;
14553	}
14554	}
14555	else
14556	{
14557	/*
14558	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14559	*/
14560	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14561	AssertRCReturn(rc2, rc2);
14562
14563	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14564	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14565
14566	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14567	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14568
14569	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14570	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14571	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14572	}
14573	return rcStrict;
14574	}
14575
14576
14577	/**
14578	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14579	*/
14580	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14581	{
14582	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14583
14584	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14585	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14586	AssertRCReturn(rc, rc);
14587
14588	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14589	return VINF_EM_RAW_EMULATE_INSTR;
14590
14591	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14592	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14593	}
14594
14595
14596	/**
14597	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14598	*/
14599	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14600	{
14601	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14602
14603	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14604	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14605	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14606	AssertRCReturn(rc, rc);
14607
14608	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
14609	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14610	{
14611	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14612	we must reset offsetting on VM-entry. See @bugref{6634}. */
14613	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14614	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14615	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14616	}
14617	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14618	{
14619	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14620	rcStrict = VINF_SUCCESS;
14621	}
14622	return rcStrict;
14623	}
14624
14625
14626	/**
14627	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14628	*/
14629	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14630	{
14631	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14632
14633	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14634	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14635	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14636	AssertRCReturn(rc, rc);
14637
14638	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
14639	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14640	{
14641	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14642	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14643	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14644	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14645	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14646	}
14647	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14648	{
14649	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14650	rcStrict = VINF_SUCCESS;
14651	}
14652	return rcStrict;
14653	}
14654
14655
14656	/**
14657	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14658	*/
14659	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14660	{
14661	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14662
14663	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14664	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14665	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14666	AssertRCReturn(rc, rc);
14667
14668	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14669	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14670	if (RT_LIKELY(rc == VINF_SUCCESS))
14671	{
14672	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14673	Assert(pVmxTransient->cbExitInstr == 2);
14674	}
14675	else
14676	{
14677	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14678	rc = VERR_EM_INTERPRETER;
14679	}
14680	return rc;
14681	}
14682
14683
14684	/**
14685	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14686	*/
14687	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14688	{
14689	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14690
14691	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14692	if (EMAreHypercallInstructionsEnabled(pVCpu))
14693	{
14694	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14695	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14696	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14697	AssertRCReturn(rc, rc);
14698
14699	/* Perform the hypercall. */
14700	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14701	if (rcStrict == VINF_SUCCESS)
14702	{
14703	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14704	AssertRCReturn(rc, rc);
14705	}
14706	else
14707	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14708	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14709	\|\| RT_FAILURE(rcStrict));
14710
14711	/* If the hypercall changes anything other than guest's general-purpose registers,
14712	we would need to reload the guest changed bits here before VM-entry. */
14713	}
14714	else
14715	Log4Func(("Hypercalls not enabled\n"));
14716
14717	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14718	if (RT_FAILURE(rcStrict))
14719	{
14720	hmR0VmxSetPendingXcptUD(pVCpu);
14721	rcStrict = VINF_SUCCESS;
14722	}
14723
14724	return rcStrict;
14725	}
14726
14727
14728	/**
14729	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14730	*/
14731	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14732	{
14733	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14734	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14735
14736	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14737	hmR0VmxReadExitQualVmcs(pVmxTransient);
14738	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14739	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14740	AssertRCReturn(rc, rc);
14741
14742	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
14743
14744	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14745	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14746	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14747	{
14748	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14749	rcStrict = VINF_SUCCESS;
14750	}
14751	else
14752	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14753	VBOXSTRICTRC_VAL(rcStrict)));
14754	return rcStrict;
14755	}
14756
14757
14758	/**
14759	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14760	*/
14761	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14762	{
14763	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14764
14765	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14766	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14767	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14768	AssertRCReturn(rc, rc);
14769
14770	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
14771	if (rcStrict == VINF_SUCCESS)
14772	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14773	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14774	{
14775	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14776	rcStrict = VINF_SUCCESS;
14777	}
14778
14779	return rcStrict;
14780	}
14781
14782
14783	/**
14784	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14785	*/
14786	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14787	{
14788	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14789
14790	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14791	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14792	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14793	AssertRCReturn(rc, rc);
14794
14795	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
14796	if (RT_SUCCESS(rcStrict))
14797	{
14798	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14799	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14800	rcStrict = VINF_SUCCESS;
14801	}
14802
14803	return rcStrict;
14804	}
14805
14806
14807	/**
14808	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14809	* VM-exit.
14810	*/
14811	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14812	{
14813	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14814	return VINF_EM_RESET;
14815	}
14816
14817
14818	/**
14819	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14820	*/
14821	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14822	{
14823	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14824
14825	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14826	AssertRCReturn(rc, rc);
14827
14828	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14829	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14830	rc = VINF_SUCCESS;
14831	else
14832	rc = VINF_EM_HALT;
14833
14834	if (rc != VINF_SUCCESS)
14835	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14836	return rc;
14837	}
14838
14839
14840	/**
14841	* VM-exit handler for instructions that result in a \#UD exception delivered to
14842	* the guest.
14843	*/
14844	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14845	{
14846	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14847	hmR0VmxSetPendingXcptUD(pVCpu);
14848	return VINF_SUCCESS;
14849	}
14850
14851
14852	/**
14853	* VM-exit handler for expiry of the VMX-preemption timer.
14854	*/
14855	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14856	{
14857	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14858
14859	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14860	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14861
14862	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14863	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14864	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14865	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14866	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14867	}
14868
14869
14870	/**
14871	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14872	*/
14873	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14874	{
14875	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14876
14877	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14878	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14879	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14880	AssertRCReturn(rc, rc);
14881
14882	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
14883	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14884	: HM_CHANGED_RAISED_XCPT_MASK);
14885
14886	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14887	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14888
14889	return rcStrict;
14890	}
14891
14892
14893	/**
14894	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14895	*/
14896	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14897	{
14898	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14899
14900	/** @todo Enable the new code after finding a reliably guest test-case. */
14901	#if 1
14902	return VERR_EM_INTERPRETER;
14903	#else
14904	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14905	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14906	hmR0VmxReadExitQualVmcs(pVmxTransient);
14907	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
14908	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14909	AssertRCReturn(rc, rc);
14910
14911	/* Paranoia. Ensure this has a memory operand. */
14912	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
14913
14914	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
14915	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14916	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
14917	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
14918
14919	RTGCPTR GCPtrDesc;
14920	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
14921
14922	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
14923	GCPtrDesc, uType);
14924	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14925	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14926	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14927	{
14928	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14929	rcStrict = VINF_SUCCESS;
14930	}
14931	return rcStrict;
14932	#endif
14933	}
14934
14935
14936	/**
14937	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14938	* VM-exit.
14939	*/
14940	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14941	{
14942	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14943	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14944	AssertRCReturn(rc, rc);
14945
14946	rc = hmR0VmxCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14947	if (RT_FAILURE(rc))
14948	return rc;
14949
14950	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14951	NOREF(uInvalidReason);
14952
14953	#ifdef VBOX_STRICT
14954	uint32_t fIntrState;
14955	uint64_t u64Val;
14956	hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14957	hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14958	hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14959
14960	Log4(("uInvalidReason %u\n", uInvalidReason));
14961	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14962	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14963	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14964
14965	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
14966	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14967	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
14968	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
14969	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
14970	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
14971	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
14972	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14973	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
14974	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
14975	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
14976	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14977	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14978	{
14979	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14980	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14981	}
14982	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
14983	#endif
14984
14985	return VERR_VMX_INVALID_GUEST_STATE;
14986	}
14987
14988	/**
14989	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14990	*/
14991	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14992	{
14993	/*
14994	* Cumulative notes of all recognized but unexpected VM-exits.
14995	*
14996	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
14997	* nested-paging is used.
14998	*
14999	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
15000	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
15001	* this function (and thereby stop VM execution) for handling such instructions.
15002	*
15003	*
15004	* VMX_EXIT_INIT_SIGNAL:
15005	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
15006	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
15007	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
15008	*
15009	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
15010	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
15011	* See Intel spec. "23.8 Restrictions on VMX operation".
15012	*
15013	* VMX_EXIT_SIPI:
15014	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
15015	* activity state is used. We don't make use of it as our guests don't have direct
15016	* access to the host local APIC.
15017	*
15018	* See Intel spec. 25.3 "Other Causes of VM-exits".
15019	*
15020	* VMX_EXIT_IO_SMI:
15021	* VMX_EXIT_SMI:
15022	* This can only happen if we support dual-monitor treatment of SMI, which can be
15023	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
15024	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
15025	* VMX root mode or receive an SMI. If we get here, something funny is going on.
15026	*
15027	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
15028	* See Intel spec. 25.3 "Other Causes of VM-Exits"
15029	*
15030	* VMX_EXIT_ERR_MSR_LOAD:
15031	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
15032	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
15033	* execution.
15034	*
15035	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
15036	*
15037	* VMX_EXIT_ERR_MACHINE_CHECK:
15038	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
15039	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
15040	* #MC exception abort class exception is raised. We thus cannot assume a
15041	* reasonable chance of continuing any sort of execution and we bail.
15042	*
15043	* See Intel spec. 15.1 "Machine-check Architecture".
15044	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
15045	*
15046	* VMX_EXIT_PML_FULL:
15047	* VMX_EXIT_VIRTUALIZED_EOI:
15048	* VMX_EXIT_APIC_WRITE:
15049	* We do not currently support any of these features and thus they are all unexpected
15050	* VM-exits.
15051	*
15052	* VMX_EXIT_GDTR_IDTR_ACCESS:
15053	* VMX_EXIT_LDTR_TR_ACCESS:
15054	* VMX_EXIT_RDRAND:
15055	* VMX_EXIT_RSM:
15056	* VMX_EXIT_VMFUNC:
15057	* VMX_EXIT_ENCLS:
15058	* VMX_EXIT_RDSEED:
15059	* VMX_EXIT_XSAVES:
15060	* VMX_EXIT_XRSTORS:
15061	* VMX_EXIT_UMWAIT:
15062	* VMX_EXIT_TPAUSE:
15063	* These VM-exits are -not- caused unconditionally by execution of the corresponding
15064	* instruction. Any VM-exit for these instructions indicate a hardware problem,
15065	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
15066	*
15067	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
15068	*/
15069	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15070	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
15071	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15072	}
15073
15074
15075	/**
15076	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
15077	*/
15078	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15079	{
15080	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15081
15082	/** @todo Optimize this: We currently drag in the whole MSR state
15083	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15084	* MSRs required. That would require changes to IEM and possibly CPUM too.
15085	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15086	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15087	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15088	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15089	switch (idMsr)
15090	{
15091	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15092	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15093	}
15094
15095	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15096	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15097	AssertRCReturn(rc, rc);
15098
15099	Log4Func(("ecx=%#RX32\n", idMsr));
15100
15101	#ifdef VBOX_STRICT
15102	Assert(!pVmxTransient->fIsNestedGuest);
15103	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15104	{
15105	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15106	&& idMsr != MSR_K6_EFER)
15107	{
15108	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15109	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15110	}
15111	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15112	{
15113	Assert(pVmcsInfo->pvMsrBitmap);
15114	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15115	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15116	{
15117	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15118	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15119	}
15120	}
15121	}
15122	#endif
15123
15124	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
15125	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15126	if (rcStrict == VINF_SUCCESS)
15127	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15128	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15129	{
15130	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15131	rcStrict = VINF_SUCCESS;
15132	}
15133	else
15134	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
15135	("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15136
15137	return rcStrict;
15138	}
15139
15140
15141	/**
15142	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15143	*/
15144	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15145	{
15146	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15147
15148	/** @todo Optimize this: We currently drag in the whole MSR state
15149	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15150	* MSRs required. That would require changes to IEM and possibly CPUM too.
15151	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15152	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15153	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15154
15155	/*
15156	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15157	* Although we don't need to fetch the base as it will be overwritten shortly, while
15158	* loading guest-state we would also load the entire segment register including limit
15159	* and attributes and thus we need to load them here.
15160	*/
15161	switch (idMsr)
15162	{
15163	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15164	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15165	}
15166
15167	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15168	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15169	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15170	AssertRCReturn(rc, rc);
15171
15172	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15173
15174	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
15175	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15176
15177	if (rcStrict == VINF_SUCCESS)
15178	{
15179	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15180
15181	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15182	if ( idMsr == MSR_IA32_APICBASE
15183	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15184	&& idMsr <= MSR_IA32_X2APIC_END))
15185	{
15186	/*
15187	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15188	* When full APIC register virtualization is implemented we'll have to make
15189	* sure APIC state is saved from the VMCS before IEM changes it.
15190	*/
15191	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15192	}
15193	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15194	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15195	else if (idMsr == MSR_K6_EFER)
15196	{
15197	/*
15198	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15199	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15200	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15201	*/
15202	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15203	}
15204
15205	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15206	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15207	{
15208	switch (idMsr)
15209	{
15210	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15211	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15212	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15213	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15214	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15215	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15216	default:
15217	{
15218	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15219	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15220	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15221	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15222	break;
15223	}
15224	}
15225	}
15226	#ifdef VBOX_STRICT
15227	else
15228	{
15229	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15230	switch (idMsr)
15231	{
15232	case MSR_IA32_SYSENTER_CS:
15233	case MSR_IA32_SYSENTER_EIP:
15234	case MSR_IA32_SYSENTER_ESP:
15235	case MSR_K8_FS_BASE:
15236	case MSR_K8_GS_BASE:
15237	{
15238	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15239	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15240	}
15241
15242	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15243	default:
15244	{
15245	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15246	{
15247	/* EFER MSR writes are always intercepted. */
15248	if (idMsr != MSR_K6_EFER)
15249	{
15250	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15251	idMsr));
15252	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15253	}
15254	}
15255
15256	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15257	{
15258	Assert(pVmcsInfo->pvMsrBitmap);
15259	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15260	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15261	{
15262	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15263	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15264	}
15265	}
15266	break;
15267	}
15268	}
15269	}
15270	#endif /* VBOX_STRICT */
15271	}
15272	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15273	{
15274	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15275	rcStrict = VINF_SUCCESS;
15276	}
15277	else
15278	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
15279	("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15280
15281	return rcStrict;
15282	}
15283
15284
15285	/**
15286	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15287	*/
15288	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15289	{
15290	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15291
15292	/** @todo The guest has likely hit a contended spinlock. We might want to
15293	* poke a schedule different guest VCPU. */
15294	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15295	if (RT_SUCCESS(rc))
15296	return VINF_EM_RAW_INTERRUPT;
15297
15298	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15299	return rc;
15300	}
15301
15302
15303	/**
15304	* VM-exit handler for when the TPR value is lowered below the specified
15305	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15306	*/
15307	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15308	{
15309	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15310	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15311
15312	/*
15313	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15314	* We'll re-evaluate pending interrupts and inject them before the next VM
15315	* entry so we can just continue execution here.
15316	*/
15317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15318	return VINF_SUCCESS;
15319	}
15320
15321
15322	/**
15323	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15324	* VM-exit.
15325	*
15326	* @retval VINF_SUCCESS when guest execution can continue.
15327	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15328	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15329	* incompatible guest state for VMX execution (real-on-v86 case).
15330	*/
15331	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15332	{
15333	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15334	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15335
15336	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15337	hmR0VmxReadExitQualVmcs(pVmxTransient);
15338	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15339
15340	VBOXSTRICTRC rcStrict;
15341	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15342	uint64_t const uExitQual = pVmxTransient->uExitQual;
15343	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15344	switch (uAccessType)
15345	{
15346	/*
15347	* MOV to CRx.
15348	*/
15349	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15350	{
15351	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15352	AssertRCReturn(rc, rc);
15353
15354	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15355	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15356	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15357	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15358
15359	/*
15360	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15361	* - When nested paging isn't used.
15362	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15363	* - We are executing in the VM debug loop.
15364	*/
15365	Assert( iCrReg != 3
15366	\|\| !pVM->hm.s.fNestedPaging
15367	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15368	\|\| pVCpu->hm.s.fUsingDebugLoop);
15369
15370	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15371	Assert( iCrReg != 8
15372	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15373
15374	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15375	AssertMsg( rcStrict == VINF_SUCCESS
15376	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15377
15378	/*
15379	* This is a kludge for handling switches back to real mode when we try to use
15380	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15381	* deal with special selector values, so we have to return to ring-3 and run
15382	* there till the selector values are V86 mode compatible.
15383	*
15384	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15385	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15386	* this function.
15387	*/
15388	if ( iCrReg == 0
15389	&& rcStrict == VINF_SUCCESS
15390	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15391	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15392	&& (uOldCr0 & X86_CR0_PE)
15393	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15394	{
15395	/** @todo Check selectors rather than returning all the time. */
15396	Assert(!pVmxTransient->fIsNestedGuest);
15397	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15398	rcStrict = VINF_EM_RESCHEDULE_REM;
15399	}
15400	break;
15401	}
15402
15403	/*
15404	* MOV from CRx.
15405	*/
15406	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15407	{
15408	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15409	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15410
15411	/*
15412	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15413	* - When nested paging isn't used.
15414	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15415	* - We are executing in the VM debug loop.
15416	*/
15417	Assert( iCrReg != 3
15418	\|\| !pVM->hm.s.fNestedPaging
15419	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15420	\|\| pVCpu->hm.s.fUsingDebugLoop);
15421
15422	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15423	Assert( iCrReg != 8
15424	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15425
15426	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15427	break;
15428	}
15429
15430	/*
15431	* CLTS (Clear Task-Switch Flag in CR0).
15432	*/
15433	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15434	{
15435	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
15436	break;
15437	}
15438
15439	/*
15440	* LMSW (Load Machine-Status Word into CR0).
15441	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15442	*/
15443	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15444	{
15445	RTGCPTR GCPtrEffDst;
15446	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
15447	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15448	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15449	if (fMemOperand)
15450	{
15451	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
15452	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15453	}
15454	else
15455	GCPtrEffDst = NIL_RTGCPTR;
15456	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15457	break;
15458	}
15459
15460	default:
15461	{
15462	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15463	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15464	}
15465	}
15466
15467	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15468	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15469	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15470
15471	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15472	NOREF(pVM);
15473	return rcStrict;
15474	}
15475
15476
15477	/**
15478	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15479	* VM-exit.
15480	*/
15481	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15482	{
15483	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15484	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15485
15486	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15487	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15488	hmR0VmxReadExitQualVmcs(pVmxTransient);
15489	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15490	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15491	\| CPUMCTX_EXTRN_EFER);
15492	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15493	AssertRCReturn(rc, rc);
15494
15495	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15496	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15497	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15498	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15499	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15500	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15501	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15502	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15503
15504	/*
15505	* Update exit history to see if this exit can be optimized.
15506	*/
15507	VBOXSTRICTRC rcStrict;
15508	PCEMEXITREC pExitRec = NULL;
15509	if ( !fGstStepping
15510	&& !fDbgStepping)
15511	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15512	!fIOString
15513	? !fIOWrite
15514	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15515	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15516	: !fIOWrite
15517	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15518	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15519	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15520	if (!pExitRec)
15521	{
15522	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15523	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15524
15525	uint32_t const cbValue = s_aIOSizes[uIOSize];
15526	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
15527	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15528	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15529	if (fIOString)
15530	{
15531	/*
15532	* INS/OUTS - I/O String instruction.
15533	*
15534	* Use instruction-information if available, otherwise fall back on
15535	* interpreting the instruction.
15536	*/
15537	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15538	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15539	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15540	if (fInsOutsInfo)
15541	{
15542	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15543	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15544	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15545	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15546	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15547	if (fIOWrite)
15548	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15549	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15550	else
15551	{
15552	/*
15553	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15554	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15555	* See Intel Instruction spec. for "INS".
15556	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15557	*/
15558	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15559	}
15560	}
15561	else
15562	rcStrict = IEMExecOne(pVCpu);
15563
15564	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15565	fUpdateRipAlready = true;
15566	}
15567	else
15568	{
15569	/*
15570	* IN/OUT - I/O instruction.
15571	*/
15572	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15573	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15574	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15575	if (fIOWrite)
15576	{
15577	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15578	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15579	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15580	&& !pCtx->eflags.Bits.u1TF)
15581	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15582	}
15583	else
15584	{
15585	uint32_t u32Result = 0;
15586	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15587	if (IOM_SUCCESS(rcStrict))
15588	{
15589	/* Save result of I/O IN instr. in AL/AX/EAX. */
15590	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15591	}
15592	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15593	&& !pCtx->eflags.Bits.u1TF)
15594	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15595	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15596	}
15597	}
15598
15599	if (IOM_SUCCESS(rcStrict))
15600	{
15601	if (!fUpdateRipAlready)
15602	{
15603	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15604	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15605	}
15606
15607	/*
15608	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15609	* while booting Fedora 17 64-bit guest.
15610	*
15611	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15612	*/
15613	if (fIOString)
15614	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15615
15616	/*
15617	* If any I/O breakpoints are armed, we need to check if one triggered
15618	* and take appropriate action.
15619	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15620	*/
15621	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15622	AssertRCReturn(rc, rc);
15623
15624	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15625	* execution engines about whether hyper BPs and such are pending. */
15626	uint32_t const uDr7 = pCtx->dr[7];
15627	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15628	&& X86_DR7_ANY_RW_IO(uDr7)
15629	&& (pCtx->cr4 & X86_CR4_DE))
15630	\|\| DBGFBpIsHwIoArmed(pVM)))
15631	{
15632	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15633
15634	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15635	VMMRZCallRing3Disable(pVCpu);
15636	HM_DISABLE_PREEMPT(pVCpu);
15637
15638	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15639
15640	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15641	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15642	{
15643	/* Raise #DB. */
15644	if (fIsGuestDbgActive)
15645	ASMSetDR6(pCtx->dr[6]);
15646	if (pCtx->dr[7] != uDr7)
15647	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15648
15649	hmR0VmxSetPendingXcptDB(pVCpu);
15650	}
15651	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15652	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15653	else if ( rcStrict2 != VINF_SUCCESS
15654	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15655	rcStrict = rcStrict2;
15656	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15657
15658	HM_RESTORE_PREEMPT();
15659	VMMRZCallRing3Enable(pVCpu);
15660	}
15661	}
15662
15663	#ifdef VBOX_STRICT
15664	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15665	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15666	Assert(!fIOWrite);
15667	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15668	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15669	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15670	Assert(fIOWrite);
15671	else
15672	{
15673	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15674	* statuses, that the VMM device and some others may return. See
15675	* IOM_SUCCESS() for guidance. */
15676	AssertMsg( RT_FAILURE(rcStrict)
15677	\|\| rcStrict == VINF_SUCCESS
15678	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15679	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15680	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15681	\|\| rcStrict == VINF_EM_RAW_TO_R3
15682	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15683	# endif
15684	}
15685	#endif
15686	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15687	}
15688	else
15689	{
15690	/*
15691	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15692	*/
15693	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15694	AssertRCReturn(rc2, rc2);
15695	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15696	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15697	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15698	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15699	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15700	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15701
15702	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15703	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15704
15705	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15706	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15707	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15708	}
15709	return rcStrict;
15710	}
15711
15712
15713	/**
15714	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15715	* VM-exit.
15716	*/
15717	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15718	{
15719	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15720
15721	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15722	hmR0VmxReadExitQualVmcs(pVmxTransient);
15723	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15724	{
15725	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15726	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15727	{
15728	uint32_t uErrCode;
15729	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15730	{
15731	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15732	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15733	}
15734	else
15735	uErrCode = 0;
15736
15737	RTGCUINTPTR GCPtrFaultAddress;
15738	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15739	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15740	else
15741	GCPtrFaultAddress = 0;
15742
15743	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15744
15745	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15746	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
15747
15748	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15749	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15750	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15751	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15752	}
15753	}
15754
15755	/* Fall back to the interpreter to emulate the task-switch. */
15756	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15757	return VERR_EM_INTERPRETER;
15758	}
15759
15760
15761	/**
15762	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15763	*/
15764	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15765	{
15766	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15767
15768	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15769	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15770	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15771	AssertRC(rc);
15772	return VINF_EM_DBG_STEPPED;
15773	}
15774
15775
15776	/**
15777	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15778	*/
15779	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15780	{
15781	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15782	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15783
15784	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15785	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15786	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15787	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15788	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15789
15790	/*
15791	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15792	*/
15793	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15794	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15795	{
15796	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15797	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15798	{
15799	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15800	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15801	}
15802	}
15803	else
15804	{
15805	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15806	return rcStrict;
15807	}
15808
15809	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15810	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15811	hmR0VmxReadExitQualVmcs(pVmxTransient);
15812	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15813	AssertRCReturn(rc, rc);
15814
15815	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15816	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15817	switch (uAccessType)
15818	{
15819	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15820	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15821	{
15822	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15823	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15824	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15825
15826	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15827	GCPhys &= PAGE_BASE_GC_MASK;
15828	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15829	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15830	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15831
15832	rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
15833	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
15834	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15835	if ( rcStrict == VINF_SUCCESS
15836	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15837	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15838	{
15839	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15840	\| HM_CHANGED_GUEST_APIC_TPR);
15841	rcStrict = VINF_SUCCESS;
15842	}
15843	break;
15844	}
15845
15846	default:
15847	{
15848	Log4Func(("uAccessType=%#x\n", uAccessType));
15849	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15850	break;
15851	}
15852	}
15853
15854	if (rcStrict != VINF_SUCCESS)
15855	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15856	return rcStrict;
15857	}
15858
15859
15860	/**
15861	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15862	* VM-exit.
15863	*/
15864	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15865	{
15866	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15867	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15868
15869	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15870	if (!pVmxTransient->fIsNestedGuest)
15871	{
15872	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15873	if (pVmxTransient->fWasGuestDebugStateActive)
15874	{
15875	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15876	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15877	}
15878
15879	if ( !pVCpu->hm.s.fSingleInstruction
15880	&& !pVmxTransient->fWasHyperDebugStateActive)
15881	{
15882	Assert(!DBGFIsStepping(pVCpu));
15883	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15884
15885	/* Don't intercept MOV DRx any more. */
15886	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15887	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15888	AssertRC(rc);
15889
15890	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15891	VMMRZCallRing3Disable(pVCpu);
15892	HM_DISABLE_PREEMPT(pVCpu);
15893
15894	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15895	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15896	Assert(CPUMIsGuestDebugStateActive(pVCpu));
15897
15898	HM_RESTORE_PREEMPT();
15899	VMMRZCallRing3Enable(pVCpu);
15900
15901	#ifdef VBOX_WITH_STATISTICS
15902	hmR0VmxReadExitQualVmcs(pVmxTransient);
15903	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15904	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15905	else
15906	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15907	#endif
15908	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15909	return VINF_SUCCESS;
15910	}
15911	}
15912
15913	/*
15914	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15915	* The EFER MSR is always up-to-date.
15916	* Update the segment registers and DR7 from the CPU.
15917	*/
15918	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15919	hmR0VmxReadExitQualVmcs(pVmxTransient);
15920	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15921	AssertRCReturn(rc, rc);
15922	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15923
15924	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15925	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15926	{
15927	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15928	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15929	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15930	if (RT_SUCCESS(rc))
15931	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15933	}
15934	else
15935	{
15936	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15937	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15938	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15939	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15940	}
15941
15942	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15943	if (RT_SUCCESS(rc))
15944	{
15945	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15946	AssertRCReturn(rc2, rc2);
15947	return VINF_SUCCESS;
15948	}
15949	return rc;
15950	}
15951
15952
15953	/**
15954	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15955	* Conditional VM-exit.
15956	*/
15957	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15958	{
15959	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15960	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15961
15962	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15963	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15964	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15965	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15966	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15967
15968	/*
15969	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15970	*/
15971	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15972	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15973	{
15974	/*
15975	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15976	* instruction emulation to inject the original event. Otherwise, injecting the original event
15977	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
15978	*/
15979	if (!pVCpu->hm.s.Event.fPending)
15980	{ /* likely */ }
15981	else
15982	{
15983	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15984	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15985	/** @todo NSTVMX: Think about how this should be handled. */
15986	if (pVmxTransient->fIsNestedGuest)
15987	return VERR_VMX_IPE_3;
15988	#endif
15989	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15990	}
15991	}
15992	else
15993	{
15994	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15995	return rcStrict;
15996	}
15997
15998	/*
15999	* Get sufficient state and update the exit history entry.
16000	*/
16001	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16002	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
16003	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16004	AssertRCReturn(rc, rc);
16005
16006	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
16007	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
16008	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
16009	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
16010	if (!pExitRec)
16011	{
16012	/*
16013	* If we succeed, resume guest execution.
16014	* If we fail in interpreting the instruction because we couldn't get the guest physical address
16015	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
16016	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
16017	* weird case. See @bugref{6043}.
16018	*/
16019	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16020	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16021	/** @todo bird: We can probably just go straight to IOM here and assume that
16022	* it's MMIO, then fall back on PGM if that hunch didn't work out so
16023	* well. However, we need to address that aliasing workarounds that
16024	* PGMR0Trap0eHandlerNPMisconfig implements. So, some care is needed.
16025	*
16026	* Might also be interesting to see if we can get this done more or
16027	* less locklessly inside IOM. Need to consider the lookup table
16028	* updating and use a bit more carefully first (or do all updates via
16029	* rendezvous) */
16030	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
16031	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
16032	if ( rcStrict == VINF_SUCCESS
16033	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16034	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16035	{
16036	/* Successfully handled MMIO operation. */
16037	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16038	\| HM_CHANGED_GUEST_APIC_TPR);
16039	rcStrict = VINF_SUCCESS;
16040	}
16041	}
16042	else
16043	{
16044	/*
16045	* Frequent exit or something needing probing. Call EMHistoryExec.
16046	*/
16047	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
16048	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
16049
16050	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16051	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16052
16053	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16054	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16055	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16056	}
16057	return rcStrict;
16058	}
16059
16060
16061	/**
16062	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
16063	* VM-exit.
16064	*/
16065	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16066	{
16067	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16068	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16069
16070	hmR0VmxReadExitQualVmcs(pVmxTransient);
16071	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16072	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16073	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16074	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16075	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16076
16077	/*
16078	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
16079	*/
16080	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16081	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16082	{
16083	/*
16084	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
16085	* we shall resolve the nested #PF and re-inject the original event.
16086	*/
16087	if (pVCpu->hm.s.Event.fPending)
16088	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
16089	}
16090	else
16091	{
16092	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
16093	return rcStrict;
16094	}
16095
16096	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16097	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
16098	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16099	AssertRCReturn(rc, rc);
16100
16101	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
16102	uint64_t const uExitQual = pVmxTransient->uExitQual;
16103	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
16104
16105	RTGCUINT uErrorCode = 0;
16106	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16107	uErrorCode \|= X86_TRAP_PF_ID;
16108	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16109	uErrorCode \|= X86_TRAP_PF_RW;
16110	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16111	uErrorCode \|= X86_TRAP_PF_P;
16112
16113	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16114	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16115	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
16116
16117	/*
16118	* Handle the pagefault trap for the nested shadow table.
16119	*/
16120	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16121	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16122	TRPMResetTrap(pVCpu);
16123
16124	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16125	if ( rcStrict == VINF_SUCCESS
16126	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16127	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16128	{
16129	/* Successfully synced our nested page tables. */
16130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16131	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16132	return VINF_SUCCESS;
16133	}
16134
16135	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16136	return rcStrict;
16137	}
16138
16139
16140	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16141	/**
16142	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16143	*/
16144	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16145	{
16146	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16147
16148	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16149	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16150	hmR0VmxReadExitQualVmcs(pVmxTransient);
16151	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16152	\| CPUMCTX_EXTRN_HWVIRT
16153	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16154	AssertRCReturn(rc, rc);
16155
16156	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16157
16158	VMXVEXITINFO ExitInfo;
16159	RT_ZERO(ExitInfo);
16160	ExitInfo.uReason = pVmxTransient->uExitReason;
16161	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16162	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16163	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16164	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16165
16166	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16167	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16168	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16169	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16170	{
16171	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16172	rcStrict = VINF_SUCCESS;
16173	}
16174	return rcStrict;
16175	}
16176
16177
16178	/**
16179	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16180	*/
16181	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16182	{
16183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16184
16185	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16186	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16187	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16188	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16189	AssertRCReturn(rc, rc);
16190
16191	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16192
16193	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16194	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
16195	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16196	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16197	{
16198	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16199	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16200	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16201	}
16202	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16203	return rcStrict;
16204	}
16205
16206
16207	/**
16208	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16209	*/
16210	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16211	{
16212	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16213
16214	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16215	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16216	hmR0VmxReadExitQualVmcs(pVmxTransient);
16217	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16218	\| CPUMCTX_EXTRN_HWVIRT
16219	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16220	AssertRCReturn(rc, rc);
16221
16222	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16223
16224	VMXVEXITINFO ExitInfo;
16225	RT_ZERO(ExitInfo);
16226	ExitInfo.uReason = pVmxTransient->uExitReason;
16227	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16228	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16229	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16230	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16231
16232	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16233	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16234	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16235	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16236	{
16237	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16238	rcStrict = VINF_SUCCESS;
16239	}
16240	return rcStrict;
16241	}
16242
16243
16244	/**
16245	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16246	*/
16247	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16248	{
16249	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16250
16251	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16252	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16253	hmR0VmxReadExitQualVmcs(pVmxTransient);
16254	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16255	\| CPUMCTX_EXTRN_HWVIRT
16256	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16257	AssertRCReturn(rc, rc);
16258
16259	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16260
16261	VMXVEXITINFO ExitInfo;
16262	RT_ZERO(ExitInfo);
16263	ExitInfo.uReason = pVmxTransient->uExitReason;
16264	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16265	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16266	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16267	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16268
16269	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16270	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16271	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16272	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16273	{
16274	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16275	rcStrict = VINF_SUCCESS;
16276	}
16277	return rcStrict;
16278	}
16279
16280
16281	/**
16282	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16283	*/
16284	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16285	{
16286	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16287
16288	/*
16289	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16290	* thus might not need to import the shadow VMCS state, it's safer just in case
16291	* code elsewhere dares look at unsynced VMCS fields.
16292	*/
16293	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16294	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16295	hmR0VmxReadExitQualVmcs(pVmxTransient);
16296	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16297	\| CPUMCTX_EXTRN_HWVIRT
16298	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16299	AssertRCReturn(rc, rc);
16300
16301	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16302
16303	VMXVEXITINFO ExitInfo;
16304	RT_ZERO(ExitInfo);
16305	ExitInfo.uReason = pVmxTransient->uExitReason;
16306	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16307	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16308	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16309	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16310	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16311
16312	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16313	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16315	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16316	{
16317	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16318	rcStrict = VINF_SUCCESS;
16319	}
16320	return rcStrict;
16321	}
16322
16323
16324	/**
16325	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16326	*/
16327	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16328	{
16329	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16330
16331	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16332	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16333	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16334	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16335	AssertRCReturn(rc, rc);
16336
16337	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16338
16339	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16340	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
16341	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16342	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16343	{
16344	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16345	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16346	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16347	}
16348	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16349	return rcStrict;
16350	}
16351
16352
16353	/**
16354	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16355	*/
16356	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16357	{
16358	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16359
16360	/*
16361	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16362	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16363	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16364	*/
16365	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16366	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16367	hmR0VmxReadExitQualVmcs(pVmxTransient);
16368	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16369	\| CPUMCTX_EXTRN_HWVIRT
16370	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16371	AssertRCReturn(rc, rc);
16372
16373	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16374
16375	VMXVEXITINFO ExitInfo;
16376	RT_ZERO(ExitInfo);
16377	ExitInfo.uReason = pVmxTransient->uExitReason;
16378	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16379	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16380	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16381	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16382	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16383
16384	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16385	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16386	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16387	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16388	{
16389	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16390	rcStrict = VINF_SUCCESS;
16391	}
16392	return rcStrict;
16393	}
16394
16395
16396	/**
16397	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16398	*/
16399	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16400	{
16401	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16402
16403	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16404	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16405	\| CPUMCTX_EXTRN_HWVIRT
16406	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16407	AssertRCReturn(rc, rc);
16408
16409	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16410
16411	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
16412	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16413	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16414	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16415	{
16416	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16417	rcStrict = VINF_SUCCESS;
16418	}
16419	return rcStrict;
16420	}
16421
16422
16423	/**
16424	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16425	*/
16426	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16427	{
16428	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16429
16430	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16431	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16432	hmR0VmxReadExitQualVmcs(pVmxTransient);
16433	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16434	\| CPUMCTX_EXTRN_HWVIRT
16435	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16436	AssertRCReturn(rc, rc);
16437
16438	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16439
16440	VMXVEXITINFO ExitInfo;
16441	RT_ZERO(ExitInfo);
16442	ExitInfo.uReason = pVmxTransient->uExitReason;
16443	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16444	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16445	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16446	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16447
16448	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16449	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16450	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16451	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16452	{
16453	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16454	rcStrict = VINF_SUCCESS;
16455	}
16456	return rcStrict;
16457	}
16458
16459
16460	/**
16461	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16462	*/
16463	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16464	{
16465	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16466
16467	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16468	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16469	hmR0VmxReadExitQualVmcs(pVmxTransient);
16470	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16471	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16472	AssertRCReturn(rc, rc);
16473
16474	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16475
16476	VMXVEXITINFO ExitInfo;
16477	RT_ZERO(ExitInfo);
16478	ExitInfo.uReason = pVmxTransient->uExitReason;
16479	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16480	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16481	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16482	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16483
16484	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16485	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16486	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16487	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16488	{
16489	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16490	rcStrict = VINF_SUCCESS;
16491	}
16492	return rcStrict;
16493	}
16494	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16495	/** @} */
16496
16497
16498	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16499	/** @name Nested-guest VM-exit handlers.
16500	* @{
16501	*/
16502	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16503	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16504	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16505
16506	/**
16507	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16508	* Conditional VM-exit.
16509	*/
16510	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16511	{
16512	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16513
16514	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16515
16516	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16517	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16518	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16519
16520	switch (uExitIntType)
16521	{
16522	/*
16523	* Physical NMIs:
16524	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16525	*/
16526	case VMX_EXIT_INT_INFO_TYPE_NMI:
16527	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16528
16529	/*
16530	* Hardware exceptions,
16531	* Software exceptions,
16532	* Privileged software exceptions:
16533	* Figure out if the exception must be delivered to the guest or the nested-guest.
16534	*/
16535	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16536	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16537	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16538	{
16539	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16540	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16541	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16542	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16543
16544	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16545	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16546	pVmxTransient->uExitIntErrorCode);
16547	if (fIntercept)
16548	{
16549	/* Exit qualification is required for debug and page-fault exceptions. */
16550	hmR0VmxReadExitQualVmcs(pVmxTransient);
16551
16552	/*
16553	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16554	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16555	* length. However, if delivery of a software interrupt, software exception or privileged
16556	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16557	*/
16558	VMXVEXITINFO ExitInfo;
16559	RT_ZERO(ExitInfo);
16560	ExitInfo.uReason = pVmxTransient->uExitReason;
16561	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16562	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16563
16564	VMXVEXITEVENTINFO ExitEventInfo;
16565	RT_ZERO(ExitEventInfo);
16566	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16567	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16568	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16569	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16570
16571	#ifdef DEBUG_ramshankar
16572	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16573	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
16574	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16575	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16576	{
16577	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
16578	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
16579	}
16580	#endif
16581	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16582	}
16583
16584	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16585	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16586	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16587	}
16588
16589	/*
16590	* Software interrupts:
16591	* VM-exits cannot be caused by software interrupts.
16592	*
16593	* External interrupts:
16594	* This should only happen when "acknowledge external interrupts on VM-exit"
16595	* control is set. However, we never set this when executing a guest or
16596	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16597	* the guest.
16598	*/
16599	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16600	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16601	default:
16602	{
16603	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16604	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16605	}
16606	}
16607	}
16608
16609
16610	/**
16611	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16612	* Unconditional VM-exit.
16613	*/
16614	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16615	{
16616	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16617	return IEMExecVmxVmexitTripleFault(pVCpu);
16618	}
16619
16620
16621	/**
16622	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16623	*/
16624	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16625	{
16626	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16627
16628	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16629	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16630	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16631	}
16632
16633
16634	/**
16635	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16636	*/
16637	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16638	{
16639	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16640
16641	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16642	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16643	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16644	}
16645
16646
16647	/**
16648	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16649	* Unconditional VM-exit.
16650	*/
16651	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16652	{
16653	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16654
16655	hmR0VmxReadExitQualVmcs(pVmxTransient);
16656	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16657	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16658	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16659
16660	VMXVEXITINFO ExitInfo;
16661	RT_ZERO(ExitInfo);
16662	ExitInfo.uReason = pVmxTransient->uExitReason;
16663	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16664	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16665
16666	VMXVEXITEVENTINFO ExitEventInfo;
16667	RT_ZERO(ExitEventInfo);
16668	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16669	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16670	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16671	}
16672
16673
16674	/**
16675	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16676	*/
16677	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16678	{
16679	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16680
16681	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16682	{
16683	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16684	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16685	}
16686	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16687	}
16688
16689
16690	/**
16691	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16692	*/
16693	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16694	{
16695	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16696
16697	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16698	{
16699	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16700	hmR0VmxReadExitQualVmcs(pVmxTransient);
16701
16702	VMXVEXITINFO ExitInfo;
16703	RT_ZERO(ExitInfo);
16704	ExitInfo.uReason = pVmxTransient->uExitReason;
16705	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16706	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16707	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16708	}
16709	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16710	}
16711
16712
16713	/**
16714	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16715	*/
16716	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16717	{
16718	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16719
16720	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16721	{
16722	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16723	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16724	}
16725	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16726	}
16727
16728
16729	/**
16730	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16731	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16732	*/
16733	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16734	{
16735	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16736
16737	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16738	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16739
16740	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16741
16742	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16743	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16744	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16745
16746	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16747	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16748	u64VmcsField &= UINT64_C(0xffffffff);
16749
16750	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16751	{
16752	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16753	hmR0VmxReadExitQualVmcs(pVmxTransient);
16754
16755	VMXVEXITINFO ExitInfo;
16756	RT_ZERO(ExitInfo);
16757	ExitInfo.uReason = pVmxTransient->uExitReason;
16758	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16759	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16760	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16761	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16762	}
16763
16764	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16765	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16766	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16767	}
16768
16769
16770	/**
16771	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16772	*/
16773	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16774	{
16775	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16776
16777	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16778	{
16779	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16780	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16781	}
16782
16783	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16784	}
16785
16786
16787	/**
16788	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16789	* Conditional VM-exit.
16790	*/
16791	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16792	{
16793	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16794
16795	hmR0VmxReadExitQualVmcs(pVmxTransient);
16796	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16797
16798	VBOXSTRICTRC rcStrict;
16799	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16800	switch (uAccessType)
16801	{
16802	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16803	{
16804	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16805	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16806	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16807	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16808
16809	bool fIntercept;
16810	switch (iCrReg)
16811	{
16812	case 0:
16813	case 4:
16814	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16815	break;
16816
16817	case 3:
16818	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16819	break;
16820
16821	case 8:
16822	fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16823	break;
16824
16825	default:
16826	fIntercept = false;
16827	break;
16828	}
16829	if (fIntercept)
16830	{
16831	VMXVEXITINFO ExitInfo;
16832	RT_ZERO(ExitInfo);
16833	ExitInfo.uReason = pVmxTransient->uExitReason;
16834	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16835	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16836	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16837	}
16838	else
16839	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16840	break;
16841	}
16842
16843	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16844	{
16845	/*
16846	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16847	* CR2 reads do not cause a VM-exit.
16848	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16849	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16850	*/
16851	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16852	if ( iCrReg == 3
16853	\|\| iCrReg == 8)
16854	{
16855	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16856	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16857	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16858	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
16859	{
16860	VMXVEXITINFO ExitInfo;
16861	RT_ZERO(ExitInfo);
16862	ExitInfo.uReason = pVmxTransient->uExitReason;
16863	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16864	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16865	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16866	}
16867	else
16868	{
16869	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16870	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16871	}
16872	}
16873	else
16874	{
16875	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16876	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16877	}
16878	break;
16879	}
16880
16881	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16882	{
16883	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16884	Assert(pVmcsNstGst);
16885	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16886	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16887	if ( (uGstHostMask & X86_CR0_TS)
16888	&& (uReadShadow & X86_CR0_TS))
16889	{
16890	VMXVEXITINFO ExitInfo;
16891	RT_ZERO(ExitInfo);
16892	ExitInfo.uReason = pVmxTransient->uExitReason;
16893	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16894	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16895	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16896	}
16897	else
16898	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
16899	break;
16900	}
16901
16902	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16903	{
16904	RTGCPTR GCPtrEffDst;
16905	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16906	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16907	if (fMemOperand)
16908	{
16909	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16910	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16911	}
16912	else
16913	GCPtrEffDst = NIL_RTGCPTR;
16914
16915	if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
16916	{
16917	VMXVEXITINFO ExitInfo;
16918	RT_ZERO(ExitInfo);
16919	ExitInfo.uReason = pVmxTransient->uExitReason;
16920	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16921	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16922	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16923	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16924	}
16925	else
16926	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
16927	break;
16928	}
16929
16930	default:
16931	{
16932	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16933	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16934	}
16935	}
16936
16937	if (rcStrict == VINF_IEM_RAISED_XCPT)
16938	{
16939	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16940	rcStrict = VINF_SUCCESS;
16941	}
16942	return rcStrict;
16943	}
16944
16945
16946	/**
16947	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16948	* Conditional VM-exit.
16949	*/
16950	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16951	{
16952	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16953
16954	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16955	{
16956	hmR0VmxReadExitQualVmcs(pVmxTransient);
16957	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16958
16959	VMXVEXITINFO ExitInfo;
16960	RT_ZERO(ExitInfo);
16961	ExitInfo.uReason = pVmxTransient->uExitReason;
16962	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16963	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16964	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16965	}
16966	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16967	}
16968
16969
16970	/**
16971	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16972	* Conditional VM-exit.
16973	*/
16974	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16975	{
16976	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16977
16978	hmR0VmxReadExitQualVmcs(pVmxTransient);
16979
16980	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16981	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16982	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16983
16984	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16985	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16986	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16987	{
16988	/*
16989	* IN/OUT instruction:
16990	* - Provides VM-exit instruction length.
16991	*
16992	* INS/OUTS instruction:
16993	* - Provides VM-exit instruction length.
16994	* - Provides Guest-linear address.
16995	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16996	*/
16997	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16998	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16999
17000	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
17001	pVmxTransient->ExitInstrInfo.u = 0;
17002	pVmxTransient->uGuestLinearAddr = 0;
17003
17004	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
17005	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
17006	if (fIOString)
17007	{
17008	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
17009	if (fVmxInsOutsInfo)
17010	{
17011	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
17012	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17013	}
17014	}
17015
17016	VMXVEXITINFO ExitInfo;
17017	RT_ZERO(ExitInfo);
17018	ExitInfo.uReason = pVmxTransient->uExitReason;
17019	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17020	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17021	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17022	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
17023	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17024	}
17025	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
17026	}
17027
17028
17029	/**
17030	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
17031	*/
17032	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17033	{
17034	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17035
17036	uint32_t fMsrpm;
17037	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17038	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17039	else
17040	fMsrpm = VMXMSRPM_EXIT_RD;
17041
17042	if (fMsrpm & VMXMSRPM_EXIT_RD)
17043	{
17044	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17045	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17046	}
17047	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
17048	}
17049
17050
17051	/**
17052	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
17053	*/
17054	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17055	{
17056	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17057
17058	uint32_t fMsrpm;
17059	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17060	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17061	else
17062	fMsrpm = VMXMSRPM_EXIT_WR;
17063
17064	if (fMsrpm & VMXMSRPM_EXIT_WR)
17065	{
17066	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17067	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17068	}
17069	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17070	}
17071
17072
17073	/**
17074	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17075	*/
17076	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17077	{
17078	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17079
17080	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17081	{
17082	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17083	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17084	}
17085	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17086	}
17087
17088
17089	/**
17090	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17091	* VM-exit.
17092	*/
17093	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17094	{
17095	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17096
17097	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17098	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
17099	VMXVEXITINFO ExitInfo;
17100	RT_ZERO(ExitInfo);
17101	ExitInfo.uReason = pVmxTransient->uExitReason;
17102	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
17103	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
17104	}
17105
17106
17107	/**
17108	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17109	*/
17110	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17111	{
17112	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17113
17114	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17115	{
17116	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17117	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17118	}
17119	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17120	}
17121
17122
17123	/**
17124	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17125	*/
17126	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17127	{
17128	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17129
17130	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17131	* PAUSE when executing a nested-guest? If it does not, we would not need
17132	* to check for the intercepts here. Just call VM-exit... */
17133
17134	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17135	if ( CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17136	\|\| CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17137	{
17138	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17139	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17140	}
17141	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17142	}
17143
17144
17145	/**
17146	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17147	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17148	*/
17149	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17150	{
17151	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17152
17153	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17154	{
17155	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
17156	VMXVEXITINFO ExitInfo;
17157	RT_ZERO(ExitInfo);
17158	ExitInfo.uReason = pVmxTransient->uExitReason;
17159	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
17160	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
17161	}
17162	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17163	}
17164
17165
17166	/**
17167	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17168	* VM-exit.
17169	*/
17170	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17171	{
17172	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17173
17174	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17175	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17176	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17177	hmR0VmxReadExitQualVmcs(pVmxTransient);
17178
17179	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17180
17181	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
17182	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
17183
17184	VMXVEXITINFO ExitInfo;
17185	RT_ZERO(ExitInfo);
17186	ExitInfo.uReason = pVmxTransient->uExitReason;
17187	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17188	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17189
17190	VMXVEXITEVENTINFO ExitEventInfo;
17191	RT_ZERO(ExitEventInfo);
17192	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17193	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17194	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17195	}
17196
17197
17198	/**
17199	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17200	* Conditional VM-exit.
17201	*/
17202	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17203	{
17204	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17205
17206	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17207	hmR0VmxReadExitQualVmcs(pVmxTransient);
17208	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17209	}
17210
17211
17212	/**
17213	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17214	* Conditional VM-exit.
17215	*/
17216	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17217	{
17218	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17219
17220	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17221	hmR0VmxReadExitQualVmcs(pVmxTransient);
17222	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17223	}
17224
17225
17226	/**
17227	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17228	*/
17229	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17230	{
17231	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17232
17233	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17234	{
17235	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17236	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17237	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17238	}
17239	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17240	}
17241
17242
17243	/**
17244	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17245	*/
17246	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17247	{
17248	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17249
17250	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17251	{
17252	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17253	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17254	}
17255	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17256	}
17257
17258
17259	/**
17260	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17261	*/
17262	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17263	{
17264	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17265
17266	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17267	{
17268	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17269	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17270	hmR0VmxReadExitQualVmcs(pVmxTransient);
17271	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17272
17273	VMXVEXITINFO ExitInfo;
17274	RT_ZERO(ExitInfo);
17275	ExitInfo.uReason = pVmxTransient->uExitReason;
17276	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17277	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17278	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17279	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17280	}
17281	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17282	}
17283
17284
17285	/**
17286	* Nested-guest VM-exit handler for invalid-guest state
17287	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17288	*/
17289	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17290	{
17291	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17292
17293	/*
17294	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17295	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17296	* Handle it like it's in an invalid guest state of the outer guest.
17297	*
17298	* When the fast path is implemented, this should be changed to cause the corresponding
17299	* nested-guest VM-exit.
17300	*/
17301	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17302	}
17303
17304
17305	/**
17306	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17307	* and only provide the instruction length.
17308	*
17309	* Unconditional VM-exit.
17310	*/
17311	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17312	{
17313	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17314
17315	#ifdef VBOX_STRICT
17316	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17317	switch (pVmxTransient->uExitReason)
17318	{
17319	case VMX_EXIT_ENCLS:
17320	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17321	break;
17322
17323	case VMX_EXIT_VMFUNC:
17324	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
17325	break;
17326	}
17327	#endif
17328
17329	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17330	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17331	}
17332
17333
17334	/**
17335	* Nested-guest VM-exit handler for instructions that provide instruction length as
17336	* well as more information.
17337	*
17338	* Unconditional VM-exit.
17339	*/
17340	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17341	{
17342	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17343
17344	#ifdef VBOX_STRICT
17345	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17346	switch (pVmxTransient->uExitReason)
17347	{
17348	case VMX_EXIT_GDTR_IDTR_ACCESS:
17349	case VMX_EXIT_LDTR_TR_ACCESS:
17350	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17351	break;
17352
17353	case VMX_EXIT_RDRAND:
17354	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17355	break;
17356
17357	case VMX_EXIT_RDSEED:
17358	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17359	break;
17360
17361	case VMX_EXIT_XSAVES:
17362	case VMX_EXIT_XRSTORS:
17363	/** @todo NSTVMX: Verify XSS-bitmap. */
17364	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17365	break;
17366
17367	case VMX_EXIT_UMWAIT:
17368	case VMX_EXIT_TPAUSE:
17369	Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17370	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17371	break;
17372	}
17373	#endif
17374
17375	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17376	hmR0VmxReadExitQualVmcs(pVmxTransient);
17377	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17378
17379	VMXVEXITINFO ExitInfo;
17380	RT_ZERO(ExitInfo);
17381	ExitInfo.uReason = pVmxTransient->uExitReason;
17382	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17383	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17384	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17385	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17386	}
17387
17388	/** @} */
17389	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17390

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 87372

Download in other formats: