HMVMXR0.cpp@ 87390

Last change on this file since 87390 was 87385, checked in by vboxsync, 4 years ago
VMM/HMVMXR0: Put VMX_RESTORE_HOST_REQUIRED at the top to simplify check for calling VMXRestoreHostState.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 722.4 KB

Line
1	/* $Id: HMVMXR0.cpp 87385 2021-01-22 23:03:58Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#include "HMInternal.h"
39	#include <VBox/vmm/vmcc.h>
40	#include <VBox/vmm/hmvmxinline.h>
41	#include "HMVMXR0.h"
42	#include "dtrace/VBoxVMM.h"
43
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
47	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/**
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
83	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
84
85	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
86	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
87	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
88	\| HMVMX_READ_EXIT_INSTR_LEN \
89	\| HMVMX_READ_IDT_VECTORING_INFO \
90	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
91
92	/** Assert that all the given fields have been read from the VMCS. */
93	#ifdef VBOX_STRICT
94	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
95	do { \
96	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
97	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
98	} while (0)
99	#else
100	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
101	#endif
102
103	/**
104	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
105	* guest using hardware-assisted VMX.
106	*
107	* This excludes state like GPRs (other than RSP) which are always are
108	* swapped and restored across the world-switch and also registers like EFER,
109	* MSR which cannot be modified by the guest without causing a VM-exit.
110	*/
111	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
112	\| CPUMCTX_EXTRN_RFLAGS \
113	\| CPUMCTX_EXTRN_RSP \
114	\| CPUMCTX_EXTRN_SREG_MASK \
115	\| CPUMCTX_EXTRN_TABLE_MASK \
116	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
117	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
118	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
119	\| CPUMCTX_EXTRN_TSC_AUX \
120	\| CPUMCTX_EXTRN_OTHER_MSRS \
121	\| CPUMCTX_EXTRN_CR0 \
122	\| CPUMCTX_EXTRN_CR3 \
123	\| CPUMCTX_EXTRN_CR4 \
124	\| CPUMCTX_EXTRN_DR7 \
125	\| CPUMCTX_EXTRN_HWVIRT \
126	\| CPUMCTX_EXTRN_HM_VMX_MASK)
127
128	/**
129	* Exception bitmap mask for real-mode guests (real-on-v86).
130	*
131	* We need to intercept all exceptions manually except:
132	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
133	* due to bugs in Intel CPUs.
134	* - \#PF need not be intercepted even in real-mode if we have nested paging
135	* support.
136	*/
137	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
138	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
139	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
140	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
141	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
142	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
143	\| RT_BIT(X86_XCPT_XF))
144
145	/** Maximum VM-instruction error number. */
146	#define HMVMX_INSTR_ERROR_MAX 28
147
148	/** Profiling macro. */
149	#ifdef HM_PROFILE_EXIT_DISPATCH
150	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
151	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
152	#else
153	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
154	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
155	#endif
156
157	/** Assert that preemption is disabled or covered by thread-context hooks. */
158	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
159	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
160
161	/** Assert that we haven't migrated CPUs when thread-context hooks are not
162	* used. */
163	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
164	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
165	("Illegal migration! Entered on CPU %u Current %u\n", \
166	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
167
168	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
169	* context. */
170	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
171	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
172	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
173
174	/** Log the VM-exit reason with an easily visible marker to identify it in a
175	* potential sea of logging data. */
176	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
177	do { \
178	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
179	HMGetVmxExitName(a_uExitReason))); \
180	} while (0) \
181
182
183	/*********************************************************************************************************************************
184	* Structures and Typedefs *
185	*********************************************************************************************************************************/
186	/**
187	* VMX per-VCPU transient state.
188	*
189	* A state structure for holding miscellaneous information across
190	* VMX non-root operation and restored after the transition.
191	*
192	* Note: The members are ordered and aligned such that the most
193	* frequently used ones (in the guest execution loop) fall within
194	* the first cache line.
195	*/
196	typedef struct VMXTRANSIENT
197	{
198	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
199	uint32_t fVmcsFieldsRead;
200	/** The guest's TPR value used for TPR shadowing. */
201	uint8_t u8GuestTpr;
202	uint8_t abAlignment0[3];
203
204	/** Whether the VM-exit was caused by a page-fault during delivery of an
205	* external interrupt or NMI. */
206	bool fVectoringPF;
207	/** Whether the VM-exit was caused by a page-fault during delivery of a
208	* contributory exception or a page-fault. */
209	bool fVectoringDoublePF;
210	/** Whether the VM-entry failed or not. */
211	bool fVMEntryFailed;
212	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
213	* area after VM-exit. */
214	bool fRemoveTscAuxMsr;
215	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
216	bool fUpdatedTscOffsettingAndPreemptTimer;
217	/** Whether we are currently executing a nested-guest. */
218	bool fIsNestedGuest;
219	/** Whether the guest debug state was active at the time of VM-exit. */
220	bool fWasGuestDebugStateActive;
221	/** Whether the hyper debug state was active at the time of VM-exit. */
222	bool fWasHyperDebugStateActive;
223
224	/** The basic VM-exit reason. */
225	uint32_t uExitReason;
226	/** The VM-exit interruption error code. */
227	uint32_t uExitIntErrorCode;
228
229	/** The host's rflags/eflags. */
230	RTCCUINTREG fEFlags;
231
232	/** The VM-exit exit code qualification. */
233	uint64_t uExitQual;
234
235	/** The VMCS info. object. */
236	PVMXVMCSINFO pVmcsInfo;
237
238	/** The VM-exit interruption-information field. */
239	uint32_t uExitIntInfo;
240	/** The VM-exit instruction-length field. */
241	uint32_t cbExitInstr;
242
243	/** The VM-exit instruction-information field. */
244	VMXEXITINSTRINFO ExitInstrInfo;
245	/** IDT-vectoring information field. */
246	uint32_t uIdtVectoringInfo;
247
248	/** IDT-vectoring error code. */
249	uint32_t uIdtVectoringErrorCode;
250	uint32_t u32Alignment0;
251
252	/** The Guest-linear address. */
253	uint64_t uGuestLinearAddr;
254
255	/** The Guest-physical address. */
256	uint64_t uGuestPhysicalAddr;
257
258	/** The Guest pending-debug exceptions. */
259	uint64_t uGuestPendingDbgXcpts;
260
261	/** The VM-entry interruption-information field. */
262	uint32_t uEntryIntInfo;
263	/** The VM-entry exception error code field. */
264	uint32_t uEntryXcptErrorCode;
265
266	/** The VM-entry instruction length field. */
267	uint32_t cbEntryInstr;
268	} VMXTRANSIENT;
269	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, fVmcsFieldsRead, 8);
271	AssertCompileMemberAlignment(VMXTRANSIENT, fVectoringPF, 8);
272	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, 8);
273	AssertCompileMemberAlignment(VMXTRANSIENT, fEFlags, 8);
274	AssertCompileMemberAlignment(VMXTRANSIENT, uExitQual, 8);
275	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, 8);
276	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, 8);
277	AssertCompileMemberAlignment(VMXTRANSIENT, ExitInstrInfo, 8);
278	AssertCompileMemberAlignment(VMXTRANSIENT, uIdtVectoringErrorCode, 8);
279	AssertCompileMemberAlignment(VMXTRANSIENT, uGuestLinearAddr, 8);
280	AssertCompileMemberAlignment(VMXTRANSIENT, uGuestPhysicalAddr, 8);
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, 8);
282	AssertCompileMemberAlignment(VMXTRANSIENT, cbEntryInstr, 8);
283	/** Pointer to VMX transient state. */
284	typedef VMXTRANSIENT *PVMXTRANSIENT;
285	/** Pointer to a const VMX transient state. */
286	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
287
288	/**
289	* VMX page allocation information.
290	*/
291	typedef struct
292	{
293	uint32_t fValid; /*< Whether to allocate this page (e.g, based on a CPU feature). /
294	uint32_t uPadding0; /*< Padding to ensure array of these structs are aligned to a multiple of 8. /
295	PRTHCPHYS pHCPhys; /*< Where to store the host-physical address of the allocation. /
296	PRTR0PTR ppVirt; /*< Where to store the host-virtual address of the allocation. /
297	} VMXPAGEALLOCINFO;
298	/** Pointer to VMX page-allocation info. */
299	typedef VMXPAGEALLOCINFO *PVMXPAGEALLOCINFO;
300	/** Pointer to a const VMX page-allocation info. */
301	typedef const VMXPAGEALLOCINFO *PCVMXPAGEALLOCINFO;
302	AssertCompileSizeAlignment(VMXPAGEALLOCINFO, 8);
303
304	/**
305	* Memory operand read or write access.
306	*/
307	typedef enum VMXMEMACCESS
308	{
309	VMXMEMACCESS_READ = 0,
310	VMXMEMACCESS_WRITE = 1
311	} VMXMEMACCESS;
312
313	/**
314	* VMX VM-exit handler.
315	*
316	* @returns Strict VBox status code (i.e. informational status codes too).
317	* @param pVCpu The cross context virtual CPU structure.
318	* @param pVmxTransient The VMX-transient structure.
319	*/
320	#ifndef HMVMX_USE_FUNCTION_TABLE
321	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
322	#else
323	typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
324	/** Pointer to VM-exit handler. */
325	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
326	#endif
327
328	/**
329	* VMX VM-exit handler, non-strict status code.
330	*
331	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
332	*
333	* @returns VBox status code, no informational status code returned.
334	* @param pVCpu The cross context virtual CPU structure.
335	* @param pVmxTransient The VMX-transient structure.
336	*
337	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
338	* use of that status code will be replaced with VINF_EM_SOMETHING
339	* later when switching over to IEM.
340	*/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
343	#else
344	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
345	#endif
346
347
348	/*********************************************************************************************************************************
349	* Internal Functions *
350	*********************************************************************************************************************************/
351	#ifndef HMVMX_USE_FUNCTION_TABLE
352	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
353	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
354	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
355	#else
356	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
357	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
358	#endif
359	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
360	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
361	#endif
362
363	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
364
365	/** @name VM-exit handler prototypes.
366	* @{
367	*/
368	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
369	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
370	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
371	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
372	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
373	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
374	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
375	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
376	static FNVMXEXITHANDLER hmR0VmxExitHlt;
377	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
378	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
379	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
380	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
381	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
382	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
383	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
384	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
385	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
386	static FNVMXEXITHANDLER hmR0VmxExitVmread;
387	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
388	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
389	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
390	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
391	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
392	#endif
393	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
394	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
395	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
396	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
397	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
398	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
399	static FNVMXEXITHANDLER hmR0VmxExitMwait;
400	static FNVMXEXITHANDLER hmR0VmxExitMtf;
401	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
402	static FNVMXEXITHANDLER hmR0VmxExitPause;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
404	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
405	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
406	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
407	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
408	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
410	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
411	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
415	/** @} */
416
417	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
418	/** @name Nested-guest VM-exit handler prototypes.
419	* @{
420	*/
421	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
422	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
423	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
424	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
425	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
426	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
427	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
428	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
429	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
430	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
431	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
432	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
433	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
434	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
435	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
436	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
437	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
438	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
439	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
440	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
441	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
442	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
443	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
444	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
445	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
446	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
447	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
448	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
449	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
450	/** @} */
451	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
452
453
454	/*********************************************************************************************************************************
455	* Global Variables *
456	*********************************************************************************************************************************/
457	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
458	/**
459	* Array of all VMCS fields.
460	* Any fields added to the VT-x spec. should be added here.
461	*
462	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
463	* of nested-guests.
464	*/
465	static const uint32_t g_aVmcsFields[] =
466	{
467	/* 16-bit control fields. */
468	VMX_VMCS16_VPID,
469	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
470	VMX_VMCS16_EPTP_INDEX,
471
472	/* 16-bit guest-state fields. */
473	VMX_VMCS16_GUEST_ES_SEL,
474	VMX_VMCS16_GUEST_CS_SEL,
475	VMX_VMCS16_GUEST_SS_SEL,
476	VMX_VMCS16_GUEST_DS_SEL,
477	VMX_VMCS16_GUEST_FS_SEL,
478	VMX_VMCS16_GUEST_GS_SEL,
479	VMX_VMCS16_GUEST_LDTR_SEL,
480	VMX_VMCS16_GUEST_TR_SEL,
481	VMX_VMCS16_GUEST_INTR_STATUS,
482	VMX_VMCS16_GUEST_PML_INDEX,
483
484	/* 16-bits host-state fields. */
485	VMX_VMCS16_HOST_ES_SEL,
486	VMX_VMCS16_HOST_CS_SEL,
487	VMX_VMCS16_HOST_SS_SEL,
488	VMX_VMCS16_HOST_DS_SEL,
489	VMX_VMCS16_HOST_FS_SEL,
490	VMX_VMCS16_HOST_GS_SEL,
491	VMX_VMCS16_HOST_TR_SEL,
492
493	/* 64-bit control fields. */
494	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
495	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
496	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
497	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
498	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
499	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
500	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
501	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
502	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
503	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
504	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
505	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
506	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
507	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
508	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
509	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
510	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
511	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
512	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
513	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
514	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
515	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
516	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
517	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
518	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
519	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
520	VMX_VMCS64_CTRL_EPTP_FULL,
521	VMX_VMCS64_CTRL_EPTP_HIGH,
522	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
523	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
524	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
525	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
526	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
527	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
528	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
529	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
530	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
531	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
532	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
533	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
534	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
535	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
536	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
537	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
538	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
539	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
540	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
541	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
542	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
543	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
544
545	/* 64-bit read-only data fields. */
546	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
547	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
548
549	/* 64-bit guest-state fields. */
550	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
551	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
552	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
553	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
554	VMX_VMCS64_GUEST_PAT_FULL,
555	VMX_VMCS64_GUEST_PAT_HIGH,
556	VMX_VMCS64_GUEST_EFER_FULL,
557	VMX_VMCS64_GUEST_EFER_HIGH,
558	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
559	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
560	VMX_VMCS64_GUEST_PDPTE0_FULL,
561	VMX_VMCS64_GUEST_PDPTE0_HIGH,
562	VMX_VMCS64_GUEST_PDPTE1_FULL,
563	VMX_VMCS64_GUEST_PDPTE1_HIGH,
564	VMX_VMCS64_GUEST_PDPTE2_FULL,
565	VMX_VMCS64_GUEST_PDPTE2_HIGH,
566	VMX_VMCS64_GUEST_PDPTE3_FULL,
567	VMX_VMCS64_GUEST_PDPTE3_HIGH,
568	VMX_VMCS64_GUEST_BNDCFGS_FULL,
569	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
570
571	/* 64-bit host-state fields. */
572	VMX_VMCS64_HOST_PAT_FULL,
573	VMX_VMCS64_HOST_PAT_HIGH,
574	VMX_VMCS64_HOST_EFER_FULL,
575	VMX_VMCS64_HOST_EFER_HIGH,
576	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
577	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
578
579	/* 32-bit control fields. */
580	VMX_VMCS32_CTRL_PIN_EXEC,
581	VMX_VMCS32_CTRL_PROC_EXEC,
582	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
583	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
584	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
585	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
586	VMX_VMCS32_CTRL_EXIT,
587	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
588	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
589	VMX_VMCS32_CTRL_ENTRY,
590	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
591	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
592	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
593	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
594	VMX_VMCS32_CTRL_TPR_THRESHOLD,
595	VMX_VMCS32_CTRL_PROC_EXEC2,
596	VMX_VMCS32_CTRL_PLE_GAP,
597	VMX_VMCS32_CTRL_PLE_WINDOW,
598
599	/* 32-bits read-only fields. */
600	VMX_VMCS32_RO_VM_INSTR_ERROR,
601	VMX_VMCS32_RO_EXIT_REASON,
602	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
603	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
604	VMX_VMCS32_RO_IDT_VECTORING_INFO,
605	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
606	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
607	VMX_VMCS32_RO_EXIT_INSTR_INFO,
608
609	/* 32-bit guest-state fields. */
610	VMX_VMCS32_GUEST_ES_LIMIT,
611	VMX_VMCS32_GUEST_CS_LIMIT,
612	VMX_VMCS32_GUEST_SS_LIMIT,
613	VMX_VMCS32_GUEST_DS_LIMIT,
614	VMX_VMCS32_GUEST_FS_LIMIT,
615	VMX_VMCS32_GUEST_GS_LIMIT,
616	VMX_VMCS32_GUEST_LDTR_LIMIT,
617	VMX_VMCS32_GUEST_TR_LIMIT,
618	VMX_VMCS32_GUEST_GDTR_LIMIT,
619	VMX_VMCS32_GUEST_IDTR_LIMIT,
620	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
621	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
622	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
623	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
624	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
625	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
626	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
627	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
628	VMX_VMCS32_GUEST_INT_STATE,
629	VMX_VMCS32_GUEST_ACTIVITY_STATE,
630	VMX_VMCS32_GUEST_SMBASE,
631	VMX_VMCS32_GUEST_SYSENTER_CS,
632	VMX_VMCS32_PREEMPT_TIMER_VALUE,
633
634	/* 32-bit host-state fields. */
635	VMX_VMCS32_HOST_SYSENTER_CS,
636
637	/* Natural-width control fields. */
638	VMX_VMCS_CTRL_CR0_MASK,
639	VMX_VMCS_CTRL_CR4_MASK,
640	VMX_VMCS_CTRL_CR0_READ_SHADOW,
641	VMX_VMCS_CTRL_CR4_READ_SHADOW,
642	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
643	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
644	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
645	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
646
647	/* Natural-width read-only data fields. */
648	VMX_VMCS_RO_EXIT_QUALIFICATION,
649	VMX_VMCS_RO_IO_RCX,
650	VMX_VMCS_RO_IO_RSI,
651	VMX_VMCS_RO_IO_RDI,
652	VMX_VMCS_RO_IO_RIP,
653	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
654
655	/* Natural-width guest-state field */
656	VMX_VMCS_GUEST_CR0,
657	VMX_VMCS_GUEST_CR3,
658	VMX_VMCS_GUEST_CR4,
659	VMX_VMCS_GUEST_ES_BASE,
660	VMX_VMCS_GUEST_CS_BASE,
661	VMX_VMCS_GUEST_SS_BASE,
662	VMX_VMCS_GUEST_DS_BASE,
663	VMX_VMCS_GUEST_FS_BASE,
664	VMX_VMCS_GUEST_GS_BASE,
665	VMX_VMCS_GUEST_LDTR_BASE,
666	VMX_VMCS_GUEST_TR_BASE,
667	VMX_VMCS_GUEST_GDTR_BASE,
668	VMX_VMCS_GUEST_IDTR_BASE,
669	VMX_VMCS_GUEST_DR7,
670	VMX_VMCS_GUEST_RSP,
671	VMX_VMCS_GUEST_RIP,
672	VMX_VMCS_GUEST_RFLAGS,
673	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
674	VMX_VMCS_GUEST_SYSENTER_ESP,
675	VMX_VMCS_GUEST_SYSENTER_EIP,
676
677	/* Natural-width host-state fields */
678	VMX_VMCS_HOST_CR0,
679	VMX_VMCS_HOST_CR3,
680	VMX_VMCS_HOST_CR4,
681	VMX_VMCS_HOST_FS_BASE,
682	VMX_VMCS_HOST_GS_BASE,
683	VMX_VMCS_HOST_TR_BASE,
684	VMX_VMCS_HOST_GDTR_BASE,
685	VMX_VMCS_HOST_IDTR_BASE,
686	VMX_VMCS_HOST_SYSENTER_ESP,
687	VMX_VMCS_HOST_SYSENTER_EIP,
688	VMX_VMCS_HOST_RSP,
689	VMX_VMCS_HOST_RIP
690	};
691	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
692
693	#ifdef VBOX_STRICT
694	static const uint32_t g_aVmcsSegBase[] =
695	{
696	VMX_VMCS_GUEST_ES_BASE,
697	VMX_VMCS_GUEST_CS_BASE,
698	VMX_VMCS_GUEST_SS_BASE,
699	VMX_VMCS_GUEST_DS_BASE,
700	VMX_VMCS_GUEST_FS_BASE,
701	VMX_VMCS_GUEST_GS_BASE
702	};
703	static const uint32_t g_aVmcsSegSel[] =
704	{
705	VMX_VMCS16_GUEST_ES_SEL,
706	VMX_VMCS16_GUEST_CS_SEL,
707	VMX_VMCS16_GUEST_SS_SEL,
708	VMX_VMCS16_GUEST_DS_SEL,
709	VMX_VMCS16_GUEST_FS_SEL,
710	VMX_VMCS16_GUEST_GS_SEL
711	};
712	static const uint32_t g_aVmcsSegLimit[] =
713	{
714	VMX_VMCS32_GUEST_ES_LIMIT,
715	VMX_VMCS32_GUEST_CS_LIMIT,
716	VMX_VMCS32_GUEST_SS_LIMIT,
717	VMX_VMCS32_GUEST_DS_LIMIT,
718	VMX_VMCS32_GUEST_FS_LIMIT,
719	VMX_VMCS32_GUEST_GS_LIMIT
720	};
721	static const uint32_t g_aVmcsSegAttr[] =
722	{
723	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
724	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
725	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
726	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
727	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
728	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
729	};
730	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
731	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
732	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
733	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
734	#endif /* VBOX_STRICT */
735
736	#ifdef HMVMX_USE_FUNCTION_TABLE
737	/**
738	* VMX_EXIT dispatch table.
739	*/
740	static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
741	{
742	/* 0 VMX_EXIT_XCPT_OR_NMI */ { hmR0VmxExitXcptOrNmi },
743	/* 1 VMX_EXIT_EXT_INT */ { hmR0VmxExitExtInt },
744	/* 2 VMX_EXIT_TRIPLE_FAULT */ { hmR0VmxExitTripleFault },
745	/* 3 VMX_EXIT_INIT_SIGNAL */ { hmR0VmxExitErrUnexpected },
746	/* 4 VMX_EXIT_SIPI */ { hmR0VmxExitErrUnexpected },
747	/* 5 VMX_EXIT_IO_SMI */ { hmR0VmxExitErrUnexpected },
748	/* 6 VMX_EXIT_SMI */ { hmR0VmxExitErrUnexpected },
749	/* 7 VMX_EXIT_INT_WINDOW */ { hmR0VmxExitIntWindow },
750	/* 8 VMX_EXIT_NMI_WINDOW */ { hmR0VmxExitNmiWindow },
751	/* 9 VMX_EXIT_TASK_SWITCH */ { hmR0VmxExitTaskSwitch },
752	/* 10 VMX_EXIT_CPUID */ { hmR0VmxExitCpuid },
753	/* 11 VMX_EXIT_GETSEC */ { hmR0VmxExitGetsec },
754	/* 12 VMX_EXIT_HLT */ { hmR0VmxExitHlt },
755	/* 13 VMX_EXIT_INVD */ { hmR0VmxExitInvd },
756	/* 14 VMX_EXIT_INVLPG */ { hmR0VmxExitInvlpg },
757	/* 15 VMX_EXIT_RDPMC */ { hmR0VmxExitRdpmc },
758	/* 16 VMX_EXIT_RDTSC */ { hmR0VmxExitRdtsc },
759	/* 17 VMX_EXIT_RSM */ { hmR0VmxExitErrUnexpected },
760	/* 18 VMX_EXIT_VMCALL */ { hmR0VmxExitVmcall },
761	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
762	/* 19 VMX_EXIT_VMCLEAR */ { hmR0VmxExitVmclear },
763	/* 20 VMX_EXIT_VMLAUNCH */ { hmR0VmxExitVmlaunch },
764	/* 21 VMX_EXIT_VMPTRLD */ { hmR0VmxExitVmptrld },
765	/* 22 VMX_EXIT_VMPTRST */ { hmR0VmxExitVmptrst },
766	/* 23 VMX_EXIT_VMREAD */ { hmR0VmxExitVmread },
767	/* 24 VMX_EXIT_VMRESUME */ { hmR0VmxExitVmresume },
768	/* 25 VMX_EXIT_VMWRITE */ { hmR0VmxExitVmwrite },
769	/* 26 VMX_EXIT_VMXOFF */ { hmR0VmxExitVmxoff },
770	/* 27 VMX_EXIT_VMXON */ { hmR0VmxExitVmxon },
771	#else
772	/* 19 VMX_EXIT_VMCLEAR */ { hmR0VmxExitSetPendingXcptUD },
773	/* 20 VMX_EXIT_VMLAUNCH */ { hmR0VmxExitSetPendingXcptUD },
774	/* 21 VMX_EXIT_VMPTRLD */ { hmR0VmxExitSetPendingXcptUD },
775	/* 22 VMX_EXIT_VMPTRST */ { hmR0VmxExitSetPendingXcptUD },
776	/* 23 VMX_EXIT_VMREAD */ { hmR0VmxExitSetPendingXcptUD },
777	/* 24 VMX_EXIT_VMRESUME */ { hmR0VmxExitSetPendingXcptUD },
778	/* 25 VMX_EXIT_VMWRITE */ { hmR0VmxExitSetPendingXcptUD },
779	/* 26 VMX_EXIT_VMXOFF */ { hmR0VmxExitSetPendingXcptUD },
780	/* 27 VMX_EXIT_VMXON */ { hmR0VmxExitSetPendingXcptUD },
781	#endif
782	/* 28 VMX_EXIT_MOV_CRX */ { hmR0VmxExitMovCRx },
783	/* 29 VMX_EXIT_MOV_DRX */ { hmR0VmxExitMovDRx },
784	/* 30 VMX_EXIT_IO_INSTR */ { hmR0VmxExitIoInstr },
785	/* 31 VMX_EXIT_RDMSR */ { hmR0VmxExitRdmsr },
786	/* 32 VMX_EXIT_WRMSR */ { hmR0VmxExitWrmsr },
787	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ { hmR0VmxExitErrInvalidGuestState },
788	/* 34 VMX_EXIT_ERR_MSR_LOAD */ { hmR0VmxExitErrUnexpected },
789	/* 35 UNDEFINED */ { hmR0VmxExitErrUnexpected },
790	/* 36 VMX_EXIT_MWAIT */ { hmR0VmxExitMwait },
791	/* 37 VMX_EXIT_MTF */ { hmR0VmxExitMtf },
792	/* 38 UNDEFINED */ { hmR0VmxExitErrUnexpected },
793	/* 39 VMX_EXIT_MONITOR */ { hmR0VmxExitMonitor },
794	/* 40 VMX_EXIT_PAUSE */ { hmR0VmxExitPause },
795	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ { hmR0VmxExitErrUnexpected },
796	/* 42 UNDEFINED */ { hmR0VmxExitErrUnexpected },
797	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ { hmR0VmxExitTprBelowThreshold },
798	/* 44 VMX_EXIT_APIC_ACCESS */ { hmR0VmxExitApicAccess },
799	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ { hmR0VmxExitErrUnexpected },
800	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ { hmR0VmxExitErrUnexpected },
801	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ { hmR0VmxExitErrUnexpected },
802	/* 48 VMX_EXIT_EPT_VIOLATION */ { hmR0VmxExitEptViolation },
803	/* 49 VMX_EXIT_EPT_MISCONFIG */ { hmR0VmxExitEptMisconfig },
804	/* 50 VMX_EXIT_INVEPT */ { hmR0VmxExitSetPendingXcptUD },
805	/* 51 VMX_EXIT_RDTSCP */ { hmR0VmxExitRdtscp },
806	/* 52 VMX_EXIT_PREEMPT_TIMER */ { hmR0VmxExitPreemptTimer },
807	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
808	/* 53 VMX_EXIT_INVVPID */ { hmR0VmxExitInvvpid },
809	#else
810	/* 53 VMX_EXIT_INVVPID */ { hmR0VmxExitSetPendingXcptUD },
811	#endif
812	/* 54 VMX_EXIT_WBINVD */ { hmR0VmxExitWbinvd },
813	/* 55 VMX_EXIT_XSETBV */ { hmR0VmxExitXsetbv },
814	/* 56 VMX_EXIT_APIC_WRITE */ { hmR0VmxExitErrUnexpected },
815	/* 57 VMX_EXIT_RDRAND */ { hmR0VmxExitErrUnexpected },
816	/* 58 VMX_EXIT_INVPCID */ { hmR0VmxExitInvpcid },
817	/* 59 VMX_EXIT_VMFUNC */ { hmR0VmxExitErrUnexpected },
818	/* 60 VMX_EXIT_ENCLS */ { hmR0VmxExitErrUnexpected },
819	/* 61 VMX_EXIT_RDSEED */ { hmR0VmxExitErrUnexpected },
820	/* 62 VMX_EXIT_PML_FULL */ { hmR0VmxExitErrUnexpected },
821	/* 63 VMX_EXIT_XSAVES */ { hmR0VmxExitErrUnexpected },
822	/* 64 VMX_EXIT_XRSTORS */ { hmR0VmxExitErrUnexpected },
823	/* 65 UNDEFINED */ { hmR0VmxExitErrUnexpected },
824	/* 66 VMX_EXIT_SPP_EVENT */ { hmR0VmxExitErrUnexpected },
825	/* 67 VMX_EXIT_UMWAIT */ { hmR0VmxExitErrUnexpected },
826	/* 68 VMX_EXIT_TPAUSE */ { hmR0VmxExitErrUnexpected },
827	};
828	#endif /* HMVMX_USE_FUNCTION_TABLE */
829
830	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
831	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
832	{
833	/* 0 */ "(Not Used)",
834	/* 1 */ "VMCALL executed in VMX root operation.",
835	/* 2 */ "VMCLEAR with invalid physical address.",
836	/* 3 */ "VMCLEAR with VMXON pointer.",
837	/* 4 */ "VMLAUNCH with non-clear VMCS.",
838	/* 5 */ "VMRESUME with non-launched VMCS.",
839	/* 6 */ "VMRESUME after VMXOFF",
840	/* 7 */ "VM-entry with invalid control fields.",
841	/* 8 */ "VM-entry with invalid host state fields.",
842	/* 9 */ "VMPTRLD with invalid physical address.",
843	/* 10 */ "VMPTRLD with VMXON pointer.",
844	/* 11 */ "VMPTRLD with incorrect revision identifier.",
845	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
846	/* 13 */ "VMWRITE to read-only VMCS component.",
847	/* 14 */ "(Not Used)",
848	/* 15 */ "VMXON executed in VMX root operation.",
849	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
850	/* 17 */ "VM-entry with non-launched executing VMCS.",
851	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
852	/* 19 */ "VMCALL with non-clear VMCS.",
853	/* 20 */ "VMCALL with invalid VM-exit control fields.",
854	/* 21 */ "(Not Used)",
855	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
856	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
857	/* 24 */ "VMCALL with invalid SMM-monitor features.",
858	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
859	/* 26 */ "VM-entry with events blocked by MOV SS.",
860	/* 27 */ "(Not Used)",
861	/* 28 */ "Invalid operand to INVEPT/INVVPID."
862	};
863	#endif /* VBOX_STRICT && LOG_ENABLED */
864
865
866	/**
867	* Checks if the given MSR is part of the lastbranch-from-IP MSR stack.
868	* @returns @c true if it's part of LBR stack, @c false otherwise.
869	*
870	* @param pVM The cross context VM structure.
871	* @param idMsr The MSR.
872	* @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
873	* Optional, can be NULL.
874	*
875	* @remarks Must only be called when LBR is enabled.
876	*/
877	DECL_FORCE_INLINE(bool) hmR0VmxIsLbrBranchFromMsr(PCVM pVM, uint32_t idMsr, uint32_t *pidxMsr)
878	{
879	Assert(pVM->hm.s.vmx.fLbr);
880	Assert(pVM->hm.s.vmx.idLbrFromIpMsrFirst);
881	uint32_t const cLbrStack = pVM->hm.s.vmx.idLbrFromIpMsrLast - pVM->hm.s.vmx.idLbrFromIpMsrFirst + 1;
882	uint32_t const idxMsr = idMsr - pVM->hm.s.vmx.idLbrFromIpMsrFirst;
883	if (idxMsr < cLbrStack)
884	{
885	if (pidxMsr)
886	*pidxMsr = idxMsr;
887	return true;
888	}
889	return false;
890	}
891
892
893	/**
894	* Checks if the given MSR is part of the lastbranch-to-IP MSR stack.
895	* @returns @c true if it's part of LBR stack, @c false otherwise.
896	*
897	* @param pVM The cross context VM structure.
898	* @param idMsr The MSR.
899	* @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
900	* Optional, can be NULL.
901	*
902	* @remarks Must only be called when LBR is enabled and when lastbranch-to-IP MSRs
903	* are supported by the CPU (see hmR0VmxSetupLbrMsrRange).
904	*/
905	DECL_FORCE_INLINE(bool) hmR0VmxIsLbrBranchToMsr(PCVM pVM, uint32_t idMsr, uint32_t *pidxMsr)
906	{
907	Assert(pVM->hm.s.vmx.fLbr);
908	if (pVM->hm.s.vmx.idLbrToIpMsrFirst)
909	{
910	uint32_t const cLbrStack = pVM->hm.s.vmx.idLbrToIpMsrLast - pVM->hm.s.vmx.idLbrToIpMsrFirst + 1;
911	uint32_t const idxMsr = idMsr - pVM->hm.s.vmx.idLbrToIpMsrFirst;
912	if (idxMsr < cLbrStack)
913	{
914	if (pidxMsr)
915	*pidxMsr = idxMsr;
916	return true;
917	}
918	}
919	return false;
920	}
921
922
923	/**
924	* Gets the CR0 guest/host mask.
925	*
926	* These bits typically does not change through the lifetime of a VM. Any bit set in
927	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
928	* by the guest.
929	*
930	* @returns The CR0 guest/host mask.
931	* @param pVCpu The cross context virtual CPU structure.
932	*/
933	static uint64_t hmR0VmxGetFixedCr0Mask(PCVMCPUCC pVCpu)
934	{
935	/*
936	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
937	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
938	*
939	* Furthermore, modifications to any bits that are reserved/unspecified currently
940	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
941	* when future CPUs specify and use currently reserved/unspecified bits.
942	*/
943	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
944	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
945	* and @bugref{6944}. */
946	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
947	return ( X86_CR0_PE
948	\| X86_CR0_NE
949	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
950	\| X86_CR0_PG
951	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
952	}
953
954
955	/**
956	* Gets the CR4 guest/host mask.
957	*
958	* These bits typically does not change through the lifetime of a VM. Any bit set in
959	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
960	* by the guest.
961	*
962	* @returns The CR4 guest/host mask.
963	* @param pVCpu The cross context virtual CPU structure.
964	*/
965	static uint64_t hmR0VmxGetFixedCr4Mask(PCVMCPUCC pVCpu)
966	{
967	/*
968	* We construct a mask of all CR4 bits that the guest can modify without causing
969	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
970	* a VM-exit when the guest attempts to modify them when executing using
971	* hardware-assisted VMX.
972	*
973	* When a feature is not exposed to the guest (and may be present on the host),
974	* we want to intercept guest modifications to the bit so we can emulate proper
975	* behavior (e.g., #GP).
976	*
977	* Furthermore, only modifications to those bits that don't require immediate
978	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
979	* depends on CR3 which might not always be the guest value while executing
980	* using hardware-assisted VMX.
981	*/
982	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
983	bool const fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
984	bool const fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
985	bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
986
987	/*
988	* Paranoia.
989	* Ensure features exposed to the guest are present on the host.
990	*/
991	Assert(!fFsGsBase \|\| pVM->cpum.ro.HostFeatures.fFsGsBase);
992	Assert(!fXSaveRstor \|\| pVM->cpum.ro.HostFeatures.fXSaveRstor);
993	Assert(!fFxSaveRstor \|\| pVM->cpum.ro.HostFeatures.fFxSaveRstor);
994
995	uint64_t const fGstMask = ( X86_CR4_PVI
996	\| X86_CR4_TSD
997	\| X86_CR4_DE
998	\| X86_CR4_MCE
999	\| X86_CR4_PCE
1000	\| X86_CR4_OSXMMEEXCPT
1001	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
1002	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
1003	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0));
1004	return ~fGstMask;
1005	}
1006
1007
1008	/**
1009	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
1010	* area.
1011	*
1012	* @returns @c true if it's different, @c false otherwise.
1013	* @param pVmcsInfo The VMCS info. object.
1014	*/
1015	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
1016	{
1017	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
1018	&& pVmcsInfo->pvGuestMsrStore);
1019	}
1020
1021
1022	/**
1023	* Sets the given Processor-based VM-execution controls.
1024	*
1025	* @param pVmxTransient The VMX-transient structure.
1026	* @param uProcCtls The Processor-based VM-execution controls to set.
1027	*/
1028	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
1029	{
1030	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1031	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
1032	{
1033	pVmcsInfo->u32ProcCtls \|= uProcCtls;
1034	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
1035	AssertRC(rc);
1036	}
1037	}
1038
1039
1040	/**
1041	* Removes the given Processor-based VM-execution controls.
1042	*
1043	* @param pVCpu The cross context virtual CPU structure.
1044	* @param pVmxTransient The VMX-transient structure.
1045	* @param uProcCtls The Processor-based VM-execution controls to remove.
1046	*
1047	* @remarks When executing a nested-guest, this will not remove any of the specified
1048	* controls if the nested hypervisor has set any one of them.
1049	*/
1050	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
1051	{
1052	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1053	if (pVmcsInfo->u32ProcCtls & uProcCtls)
1054	{
1055	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1056	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
1057	? true
1058	: !CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uProcCtls);
1059	#else
1060	NOREF(pVCpu);
1061	bool const fRemoveCtls = true;
1062	#endif
1063	if (fRemoveCtls)
1064	{
1065	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
1066	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
1067	AssertRC(rc);
1068	}
1069	}
1070	}
1071
1072
1073	/**
1074	* Sets the TSC offset for the current VMCS.
1075	*
1076	* @param uTscOffset The TSC offset to set.
1077	* @param pVmcsInfo The VMCS info. object.
1078	*/
1079	static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
1080	{
1081	if (pVmcsInfo->u64TscOffset != uTscOffset)
1082	{
1083	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
1084	AssertRC(rc);
1085	pVmcsInfo->u64TscOffset = uTscOffset;
1086	}
1087	}
1088
1089
1090	/**
1091	* Adds one or more exceptions to the exception bitmap and commits it to the current
1092	* VMCS.
1093	*
1094	* @param pVmxTransient The VMX-transient structure.
1095	* @param uXcptMask The exception(s) to add.
1096	*/
1097	static void hmR0VmxAddXcptInterceptMask(PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1098	{
1099	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1100	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1101	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1102	{
1103	uXcptBitmap \|= uXcptMask;
1104	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1105	AssertRC(rc);
1106	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1107	}
1108	}
1109
1110
1111	/**
1112	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1113	*
1114	* @param pVmxTransient The VMX-transient structure.
1115	* @param uXcpt The exception to add.
1116	*/
1117	static void hmR0VmxAddXcptIntercept(PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1118	{
1119	Assert(uXcpt <= X86_XCPT_LAST);
1120	hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1121	}
1122
1123
1124	/**
1125	* Remove one or more exceptions from the exception bitmap and commits it to the
1126	* current VMCS.
1127	*
1128	* This takes care of not removing the exception intercept if a nested-guest
1129	* requires the exception to be intercepted.
1130	*
1131	* @returns VBox status code.
1132	* @param pVCpu The cross context virtual CPU structure.
1133	* @param pVmxTransient The VMX-transient structure.
1134	* @param uXcptMask The exception(s) to remove.
1135	*/
1136	static int hmR0VmxRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1137	{
1138	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1139	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1140	if (u32XcptBitmap & uXcptMask)
1141	{
1142	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1143	if (!pVmxTransient->fIsNestedGuest)
1144	{ /* likely */ }
1145	else
1146	{
1147	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1148	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1149	}
1150	#endif
1151	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1152	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1153	\| RT_BIT(X86_XCPT_DE)
1154	\| RT_BIT(X86_XCPT_NM)
1155	\| RT_BIT(X86_XCPT_TS)
1156	\| RT_BIT(X86_XCPT_UD)
1157	\| RT_BIT(X86_XCPT_NP)
1158	\| RT_BIT(X86_XCPT_SS)
1159	\| RT_BIT(X86_XCPT_GP)
1160	\| RT_BIT(X86_XCPT_PF)
1161	\| RT_BIT(X86_XCPT_MF));
1162	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1163	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1164	#endif
1165	if (uXcptMask)
1166	{
1167	/* Validate we are not removing any essential exception intercepts. */
1168	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1169	NOREF(pVCpu);
1170	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1171	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1172
1173	/* Remove it from the exception bitmap. */
1174	u32XcptBitmap &= ~uXcptMask;
1175
1176	/* Commit and update the cache if necessary. */
1177	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1178	{
1179	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1180	AssertRC(rc);
1181	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1182	}
1183	}
1184	}
1185	return VINF_SUCCESS;
1186	}
1187
1188
1189	/**
1190	* Remove an exceptions from the exception bitmap and commits it to the current
1191	* VMCS.
1192	*
1193	* @returns VBox status code.
1194	* @param pVCpu The cross context virtual CPU structure.
1195	* @param pVmxTransient The VMX-transient structure.
1196	* @param uXcpt The exception to remove.
1197	*/
1198	static int hmR0VmxRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1199	{
1200	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1201	}
1202
1203
1204	/**
1205	* Loads the VMCS specified by the VMCS info. object.
1206	*
1207	* @returns VBox status code.
1208	* @param pVmcsInfo The VMCS info. object.
1209	*
1210	* @remarks Can be called with interrupts disabled.
1211	*/
1212	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1213	{
1214	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1215	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1216
1217	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1218	if (RT_SUCCESS(rc))
1219	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1220	return rc;
1221	}
1222
1223
1224	/**
1225	* Clears the VMCS specified by the VMCS info. object.
1226	*
1227	* @returns VBox status code.
1228	* @param pVmcsInfo The VMCS info. object.
1229	*
1230	* @remarks Can be called with interrupts disabled.
1231	*/
1232	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1233	{
1234	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1235	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1236
1237	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1238	if (RT_SUCCESS(rc))
1239	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1240	return rc;
1241	}
1242
1243
1244	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1245	/**
1246	* Loads the shadow VMCS specified by the VMCS info. object.
1247	*
1248	* @returns VBox status code.
1249	* @param pVmcsInfo The VMCS info. object.
1250	*
1251	* @remarks Can be called with interrupts disabled.
1252	*/
1253	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1254	{
1255	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1256	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1257
1258	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1259	if (RT_SUCCESS(rc))
1260	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1261	return rc;
1262	}
1263
1264
1265	/**
1266	* Clears the shadow VMCS specified by the VMCS info. object.
1267	*
1268	* @returns VBox status code.
1269	* @param pVmcsInfo The VMCS info. object.
1270	*
1271	* @remarks Can be called with interrupts disabled.
1272	*/
1273	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1274	{
1275	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1276	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1277
1278	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1279	if (RT_SUCCESS(rc))
1280	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1281	return rc;
1282	}
1283
1284
1285	/**
1286	* Switches from and to the specified VMCSes.
1287	*
1288	* @returns VBox status code.
1289	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1290	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1291	*
1292	* @remarks Called with interrupts disabled.
1293	*/
1294	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1295	{
1296	/*
1297	* Clear the VMCS we are switching out if it has not already been cleared.
1298	* This will sync any CPU internal data back to the VMCS.
1299	*/
1300	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1301	{
1302	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1303	if (RT_SUCCESS(rc))
1304	{
1305	/*
1306	* The shadow VMCS, if any, would not be active at this point since we
1307	* would have cleared it while importing the virtual hardware-virtualization
1308	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1309	* clear the shadow VMCS here, just assert for safety.
1310	*/
1311	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1312	}
1313	else
1314	return rc;
1315	}
1316
1317	/*
1318	* Clear the VMCS we are switching to if it has not already been cleared.
1319	* This will initialize the VMCS launch state to "clear" required for loading it.
1320	*
1321	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1322	*/
1323	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1324	{
1325	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1326	if (RT_SUCCESS(rc))
1327	{ /* likely */ }
1328	else
1329	return rc;
1330	}
1331
1332	/*
1333	* Finally, load the VMCS we are switching to.
1334	*/
1335	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1336	}
1337
1338
1339	/**
1340	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1341	* caller.
1342	*
1343	* @returns VBox status code.
1344	* @param pVCpu The cross context virtual CPU structure.
1345	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1346	* true) or guest VMCS (pass false).
1347	*/
1348	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1349	{
1350	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1351	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1352
1353	PVMXVMCSINFO pVmcsInfoFrom;
1354	PVMXVMCSINFO pVmcsInfoTo;
1355	if (fSwitchToNstGstVmcs)
1356	{
1357	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1358	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1359	}
1360	else
1361	{
1362	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1363	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1364	}
1365
1366	/*
1367	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1368	* preemption hook code path acquires the current VMCS.
1369	*/
1370	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1371
1372	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1373	if (RT_SUCCESS(rc))
1374	{
1375	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1376
1377	/*
1378	* If we are switching to a VMCS that was executed on a different host CPU or was
1379	* never executed before, flag that we need to export the host state before executing
1380	* guest/nested-guest code using hardware-assisted VMX.
1381	*
1382	* This could probably be done in a preemptible context since the preemption hook
1383	* will flag the necessary change in host context. However, since preemption is
1384	* already disabled and to avoid making assumptions about host specific code in
1385	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1386	* disabled.
1387	*/
1388	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1389	{ /* likely */ }
1390	else
1391	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1392
1393	ASMSetFlags(fEFlags);
1394
1395	/*
1396	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1397	* flag that we need to update the host MSR values there. Even if we decide in the
1398	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1399	* if its content differs, we would have to update the host MSRs anyway.
1400	*/
1401	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1402	}
1403	else
1404	ASMSetFlags(fEFlags);
1405	return rc;
1406	}
1407	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1408
1409
1410	/**
1411	* Updates the VM's last error record.
1412	*
1413	* If there was a VMX instruction error, reads the error data from the VMCS and
1414	* updates VCPU's last error record as well.
1415	*
1416	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1417	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1418	* VERR_VMX_INVALID_VMCS_FIELD.
1419	* @param rc The error code.
1420	*/
1421	static void hmR0VmxUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
1422	{
1423	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1424	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1425	{
1426	AssertPtrReturnVoid(pVCpu);
1427	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1428	}
1429	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1430	}
1431
1432
1433	#ifdef VBOX_STRICT
1434	/**
1435	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1436	* transient structure.
1437	*
1438	* @param pVmxTransient The VMX-transient structure.
1439	*/
1440	DECLINLINE(void) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1441	{
1442	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1443	AssertRC(rc);
1444	}
1445
1446
1447	/**
1448	* Reads the VM-entry exception error code field from the VMCS into
1449	* the VMX transient structure.
1450	*
1451	* @param pVmxTransient The VMX-transient structure.
1452	*/
1453	DECLINLINE(void) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1454	{
1455	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1456	AssertRC(rc);
1457	}
1458
1459
1460	/**
1461	* Reads the VM-entry exception error code field from the VMCS into
1462	* the VMX transient structure.
1463	*
1464	* @param pVmxTransient The VMX-transient structure.
1465	*/
1466	DECLINLINE(void) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1467	{
1468	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1469	AssertRC(rc);
1470	}
1471	#endif /* VBOX_STRICT */
1472
1473
1474	/**
1475	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1476	* transient structure.
1477	*
1478	* @param pVmxTransient The VMX-transient structure.
1479	*/
1480	DECLINLINE(void) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1481	{
1482	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1483	{
1484	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1485	AssertRC(rc);
1486	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1487	}
1488	}
1489
1490
1491	/**
1492	* Reads the VM-exit interruption error code from the VMCS into the VMX
1493	* transient structure.
1494	*
1495	* @param pVmxTransient The VMX-transient structure.
1496	*/
1497	DECLINLINE(void) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1498	{
1499	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1500	{
1501	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1502	AssertRC(rc);
1503	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1504	}
1505	}
1506
1507
1508	/**
1509	* Reads the VM-exit instruction length field from the VMCS into the VMX
1510	* transient structure.
1511	*
1512	* @param pVmxTransient The VMX-transient structure.
1513	*/
1514	DECLINLINE(void) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1515	{
1516	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1517	{
1518	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1519	AssertRC(rc);
1520	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1521	}
1522	}
1523
1524
1525	/**
1526	* Reads the VM-exit instruction-information field from the VMCS into
1527	* the VMX transient structure.
1528	*
1529	* @param pVmxTransient The VMX-transient structure.
1530	*/
1531	DECLINLINE(void) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1532	{
1533	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1534	{
1535	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1536	AssertRC(rc);
1537	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1538	}
1539	}
1540
1541
1542	/**
1543	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1544	*
1545	* @param pVmxTransient The VMX-transient structure.
1546	*/
1547	DECLINLINE(void) hmR0VmxReadExitQualVmcs(PVMXTRANSIENT pVmxTransient)
1548	{
1549	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1550	{
1551	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1552	AssertRC(rc);
1553	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1554	}
1555	}
1556
1557
1558	/**
1559	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1560	*
1561	* @param pVmxTransient The VMX-transient structure.
1562	*/
1563	DECLINLINE(void) hmR0VmxReadGuestLinearAddrVmcs(PVMXTRANSIENT pVmxTransient)
1564	{
1565	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1566	{
1567	int rc = VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1568	AssertRC(rc);
1569	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1570	}
1571	}
1572
1573
1574	/**
1575	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1576	*
1577	* @param pVmxTransient The VMX-transient structure.
1578	*/
1579	DECLINLINE(void) hmR0VmxReadGuestPhysicalAddrVmcs(PVMXTRANSIENT pVmxTransient)
1580	{
1581	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1582	{
1583	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1584	AssertRC(rc);
1585	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1586	}
1587	}
1588
1589	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1590	/**
1591	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1592	* structure.
1593	*
1594	* @param pVmxTransient The VMX-transient structure.
1595	*/
1596	DECLINLINE(void) hmR0VmxReadGuestPendingDbgXctps(PVMXTRANSIENT pVmxTransient)
1597	{
1598	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1599	{
1600	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1601	AssertRC(rc);
1602	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1603	}
1604	}
1605	#endif
1606
1607	/**
1608	* Reads the IDT-vectoring information field from the VMCS into the VMX
1609	* transient structure.
1610	*
1611	* @param pVmxTransient The VMX-transient structure.
1612	*
1613	* @remarks No-long-jump zone!!!
1614	*/
1615	DECLINLINE(void) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1616	{
1617	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1618	{
1619	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1620	AssertRC(rc);
1621	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1622	}
1623	}
1624
1625
1626	/**
1627	* Reads the IDT-vectoring error code from the VMCS into the VMX
1628	* transient structure.
1629	*
1630	* @param pVmxTransient The VMX-transient structure.
1631	*/
1632	DECLINLINE(void) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1633	{
1634	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1635	{
1636	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1637	AssertRC(rc);
1638	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1639	}
1640	}
1641
1642	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1643	/**
1644	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1645	*
1646	* @param pVmxTransient The VMX-transient structure.
1647	*/
1648	static void hmR0VmxReadAllRoFieldsVmcs(PVMXTRANSIENT pVmxTransient)
1649	{
1650	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1651	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1652	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1653	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1654	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1655	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1656	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1657	rc \|= VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1658	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1659	AssertRC(rc);
1660	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1661	\| HMVMX_READ_EXIT_INSTR_LEN
1662	\| HMVMX_READ_EXIT_INSTR_INFO
1663	\| HMVMX_READ_IDT_VECTORING_INFO
1664	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1665	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1666	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1667	\| HMVMX_READ_GUEST_LINEAR_ADDR
1668	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1669	}
1670	#endif
1671
1672	/**
1673	* Enters VMX root mode operation on the current CPU.
1674	*
1675	* @returns VBox status code.
1676	* @param pHostCpu The HM physical-CPU structure.
1677	* @param pVM The cross context VM structure. Can be
1678	* NULL, after a resume.
1679	* @param HCPhysCpuPage Physical address of the VMXON region.
1680	* @param pvCpuPage Pointer to the VMXON region.
1681	*/
1682	static int hmR0VmxEnterRootMode(PHMPHYSCPU pHostCpu, PVMCC pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1683	{
1684	Assert(pHostCpu);
1685	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1686	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1687	Assert(pvCpuPage);
1688	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690	if (pVM)
1691	{
1692	/* Write the VMCS revision identifier to the VMXON region. */
1693	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1694	}
1695
1696	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1697	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1698
1699	/* Enable the VMX bit in CR4 if necessary. */
1700	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1701
1702	/* Record whether VMXE was already prior to us enabling it above. */
1703	pHostCpu->fVmxeAlreadyEnabled = RT_BOOL(uOldCr4 & X86_CR4_VMXE);
1704
1705	/* Enter VMX root mode. */
1706	int rc = VMXEnable(HCPhysCpuPage);
1707	if (RT_FAILURE(rc))
1708	{
1709	/* Restore CR4.VMXE if it was not set prior to our attempt to set it above. */
1710	if (!pHostCpu->fVmxeAlreadyEnabled)
1711	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1712
1713	if (pVM)
1714	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1715	}
1716
1717	/* Restore interrupts. */
1718	ASMSetFlags(fEFlags);
1719	return rc;
1720	}
1721
1722
1723	/**
1724	* Exits VMX root mode operation on the current CPU.
1725	*
1726	* @returns VBox status code.
1727	* @param pHostCpu The HM physical-CPU structure.
1728	*/
1729	static int hmR0VmxLeaveRootMode(PHMPHYSCPU pHostCpu)
1730	{
1731	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1732
1733	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1734	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1735
1736	/* If we're for some reason not in VMX root mode, then don't leave it. */
1737	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1738
1739	int rc;
1740	if (uHostCr4 & X86_CR4_VMXE)
1741	{
1742	/* Exit VMX root mode and clear the VMX bit in CR4. */
1743	VMXDisable();
1744
1745	/* Clear CR4.VMXE only if it was clear prior to use setting it. */
1746	if (!pHostCpu->fVmxeAlreadyEnabled)
1747	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1748
1749	rc = VINF_SUCCESS;
1750	}
1751	else
1752	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1753
1754	/* Restore interrupts. */
1755	ASMSetFlags(fEFlags);
1756	return rc;
1757	}
1758
1759
1760	/**
1761	* Allocates pages specified as specified by an array of VMX page allocation info
1762	* objects.
1763	*
1764	* The pages contents are zero'd after allocation.
1765	*
1766	* @returns VBox status code.
1767	* @param hMemObj The ring-0 memory object associated with the allocation.
1768	* @param paAllocInfo The pointer to the first element of the VMX
1769	* page-allocation info object array.
1770	* @param cEntries The number of elements in the @a paAllocInfo array.
1771	*/
1772	static int hmR0VmxPagesAllocZ(RTR0MEMOBJ hMemObj, PVMXPAGEALLOCINFO paAllocInfo, uint32_t cEntries)
1773	{
1774	/* Figure out how many pages to allocate. */
1775	uint32_t cPages = 0;
1776	for (uint32_t iPage = 0; iPage < cEntries; iPage++)
1777	cPages += !!paAllocInfo[iPage].fValid;
1778
1779	/* Allocate the pages. */
1780	if (cPages)
1781	{
1782	size_t const cbPages = cPages << X86_PAGE_4K_SHIFT;
1783	int rc = RTR0MemObjAllocPage(&hMemObj, cbPages, false /* fExecutable */);
1784	if (RT_FAILURE(rc))
1785	return rc;
1786
1787	/* Zero the contents and assign each page to the corresponding VMX page-allocation entry. */
1788	void *pvFirstPage = RTR0MemObjAddress(hMemObj);
1789	ASMMemZero32(pvFirstPage, cbPages);
1790
1791	uint32_t iPage = 0;
1792	for (uint32_t i = 0; i < cEntries; i++)
1793	if (paAllocInfo[i].fValid)
1794	{
1795	RTHCPHYS const HCPhysPage = RTR0MemObjGetPagePhysAddr(hMemObj, iPage);
1796	void pvPage = (void )((uintptr_t)pvFirstPage + (iPage << X86_PAGE_4K_SHIFT));
1797	Assert(HCPhysPage && HCPhysPage != NIL_RTHCPHYS);
1798	AssertPtr(pvPage);
1799
1800	Assert(paAllocInfo[iPage].pHCPhys);
1801	Assert(paAllocInfo[iPage].ppVirt);
1802	*paAllocInfo[iPage].pHCPhys = HCPhysPage;
1803	*paAllocInfo[iPage].ppVirt = pvPage;
1804
1805	/* Move to next page. */
1806	++iPage;
1807	}
1808
1809	/* Make sure all valid (requested) pages have been assigned. */
1810	Assert(iPage == cPages);
1811	}
1812	return VINF_SUCCESS;
1813	}
1814
1815
1816	/**
1817	* Frees pages allocated using hmR0VmxPagesAllocZ.
1818	*
1819	* @param hMemObj The ring-0 memory object associated with the allocation.
1820	*/
1821	DECL_FORCE_INLINE(void) hmR0VmxPagesFree(RTR0MEMOBJ hMemObj)
1822	{
1823	/* We can cleanup wholesale since it's all one allocation. */
1824	RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
1825	}
1826
1827
1828	/**
1829	* Initializes a VMCS info. object.
1830	*
1831	* @param pVmcsInfo The VMCS info. object.
1832	*/
1833	static void hmR0VmxVmcsInfoInit(PVMXVMCSINFO pVmcsInfo)
1834	{
1835	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1836
1837	Assert(pVmcsInfo->hMemObj == NIL_RTR0MEMOBJ);
1838	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1839	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1840	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1841	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1842	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1843	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1844	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1845	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1846	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1847	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
1848	pVmcsInfo->idHostCpuExec = NIL_RTCPUID;
1849	}
1850
1851
1852	/**
1853	* Frees the VT-x structures for a VMCS info. object.
1854	*
1855	* @param pVmcsInfo The VMCS info. object.
1856	*/
1857	static void hmR0VmxVmcsInfoFree(PVMXVMCSINFO pVmcsInfo)
1858	{
1859	if (pVmcsInfo->hMemObj != NIL_RTR0MEMOBJ)
1860	{
1861	hmR0VmxPagesFree(pVmcsInfo->hMemObj);
1862	hmR0VmxVmcsInfoInit(pVmcsInfo);
1863	}
1864	}
1865
1866
1867	/**
1868	* Allocates the VT-x structures for a VMCS info. object.
1869	*
1870	* @returns VBox status code.
1871	* @param pVCpu The cross context virtual CPU structure.
1872	* @param pVmcsInfo The VMCS info. object.
1873	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1874	*
1875	* @remarks The caller is expected to take care of any and all allocation failures.
1876	* This function will not perform any cleanup for failures half-way
1877	* through.
1878	*/
1879	static int hmR0VmxAllocVmcsInfo(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1880	{
1881	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1882
1883	bool const fMsrBitmaps = RT_BOOL(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS);
1884	bool const fShadowVmcs = !fIsNstGstVmcs ? pVM->hm.s.vmx.fUseVmcsShadowing : pVM->cpum.ro.GuestFeatures.fVmxVmcsShadowing;
1885	Assert(!pVM->cpum.ro.GuestFeatures.fVmxVmcsShadowing); /* VMCS shadowing is not yet exposed to the guest. */
1886	VMXPAGEALLOCINFO aAllocInfo[] = {
1887	{ true, 0 /* Unused */, &pVmcsInfo->HCPhysVmcs, &pVmcsInfo->pvVmcs },
1888	{ true, 0 /* Unused */, &pVmcsInfo->HCPhysGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad },
1889	{ true, 0 /* Unused */, &pVmcsInfo->HCPhysHostMsrLoad, &pVmcsInfo->pvHostMsrLoad },
1890	{ fMsrBitmaps, 0 /* Unused */, &pVmcsInfo->HCPhysMsrBitmap, &pVmcsInfo->pvMsrBitmap },
1891	{ fShadowVmcs, 0 /* Unused */, &pVmcsInfo->HCPhysShadowVmcs, &pVmcsInfo->pvShadowVmcs },
1892	};
1893
1894	int rc = hmR0VmxPagesAllocZ(pVmcsInfo->hMemObj, &aAllocInfo[0], RT_ELEMENTS(aAllocInfo));
1895	if (RT_FAILURE(rc))
1896	return rc;
1897
1898	/*
1899	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1900	* Because they contain a symmetric list of guest MSRs to load on VM-entry and store on VM-exit.
1901	*/
1902	AssertCompile(RT_ELEMENTS(aAllocInfo) > 0);
1903	Assert(pVmcsInfo->HCPhysGuestMsrLoad != NIL_RTHCPHYS);
1904	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1905	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1906
1907	/*
1908	* Get the virtual-APIC page rather than allocating them again.
1909	*/
1910	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1911	{
1912	if (!fIsNstGstVmcs)
1913	{
1914	if (PDMHasApic(pVM))
1915	{
1916	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1917	if (RT_FAILURE(rc))
1918	return rc;
1919	Assert(pVmcsInfo->pbVirtApic);
1920	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1921	}
1922	}
1923	else
1924	{
1925	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(&pVCpu->cpum.GstCtx, &pVmcsInfo->HCPhysVirtApic);
1926	Assert(pVmcsInfo->pbVirtApic);
1927	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1928	}
1929	}
1930
1931	return VINF_SUCCESS;
1932	}
1933
1934
1935	/**
1936	* Free all VT-x structures for the VM.
1937	*
1938	* @returns IPRT status code.
1939	* @param pVM The cross context VM structure.
1940	*/
1941	static void hmR0VmxStructsFree(PVMCC pVM)
1942	{
1943	hmR0VmxPagesFree(pVM->hm.s.vmx.hMemObj);
1944	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1945	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1946	{
1947	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1948	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1949	}
1950	#endif
1951
1952	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1953	{
1954	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1955	hmR0VmxVmcsInfoFree(&pVCpu->hm.s.vmx.VmcsInfo);
1956	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1957	if (pVM->cpum.ro.GuestFeatures.fVmx)
1958	hmR0VmxVmcsInfoFree(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1959	#endif
1960	}
1961	}
1962
1963
1964	/**
1965	* Allocate all VT-x structures for the VM.
1966	*
1967	* @returns IPRT status code.
1968	* @param pVM The cross context VM structure.
1969	*
1970	* @remarks This functions will cleanup on memory allocation failures.
1971	*/
1972	static int hmR0VmxStructsAlloc(PVMCC pVM)
1973	{
1974	/*
1975	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1976	* The VMCS size cannot be more than 4096 bytes.
1977	*
1978	* See Intel spec. Appendix A.1 "Basic VMX Information".
1979	*/
1980	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1981	if (cbVmcs <= X86_PAGE_4K_SIZE)
1982	{ /* likely */ }
1983	else
1984	{
1985	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1986	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1987	}
1988
1989	/*
1990	* Allocate per-VM VT-x structures.
1991	*/
1992	bool const fVirtApicAccess = RT_BOOL(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
1993	bool const fUseVmcsShadowing = pVM->hm.s.vmx.fUseVmcsShadowing;
1994	VMXPAGEALLOCINFO aAllocInfo[] = {
1995	{ fVirtApicAccess, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess },
1996	{ fUseVmcsShadowing, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap },
1997	{ fUseVmcsShadowing, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap },
1998	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1999	{ true, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysScratch, &(PRTR0PTR)pVM->hm.s.vmx.pbScratch },
2000	#endif
2001	};
2002
2003	int rc = hmR0VmxPagesAllocZ(pVM->hm.s.vmx.hMemObj, &aAllocInfo[0], RT_ELEMENTS(aAllocInfo));
2004	if (RT_FAILURE(rc))
2005	goto cleanup;
2006
2007	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2008	/* Allocate the shadow VMCS-fields array. */
2009	if (fUseVmcsShadowing)
2010	{
2011	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
2012	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
2013	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
2014	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
2015	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
2016	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
2017	{ /* likely */ }
2018	else
2019	{
2020	rc = VERR_NO_MEMORY;
2021	goto cleanup;
2022	}
2023	}
2024	#endif
2025
2026	/*
2027	* Allocate per-VCPU VT-x structures.
2028	*/
2029	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2030	{
2031	/* Allocate the guest VMCS structures. */
2032	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
2033	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2034	if (RT_FAILURE(rc))
2035	goto cleanup;
2036
2037	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2038	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2039	if (pVM->cpum.ro.GuestFeatures.fVmx)
2040	{
2041	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2042	if (RT_FAILURE(rc))
2043	goto cleanup;
2044	}
2045	#endif
2046	}
2047
2048	return VINF_SUCCESS;
2049
2050	cleanup:
2051	hmR0VmxStructsFree(pVM);
2052	Assert(rc != VINF_SUCCESS);
2053	return rc;
2054	}
2055
2056
2057	/**
2058	* Pre-initializes non-zero fields in VMX structures that will be allocated.
2059	*
2060	* @param pVM The cross context VM structure.
2061	*/
2062	static void hmR0VmxStructsInit(PVMCC pVM)
2063	{
2064	/* Paranoia. */
2065	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
2066	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2067	Assert(pVM->hm.s.vmx.pbScratch == NULL);
2068	#endif
2069
2070	/*
2071	* Initialize members up-front so we can cleanup en masse on allocation failures.
2072	*/
2073	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2074	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
2075	#endif
2076	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
2077	pVM->hm.s.vmx.HCPhysVmreadBitmap = NIL_RTHCPHYS;
2078	pVM->hm.s.vmx.HCPhysVmwriteBitmap = NIL_RTHCPHYS;
2079	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2080	{
2081	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
2082	hmR0VmxVmcsInfoInit(&pVCpu->hm.s.vmx.VmcsInfo);
2083	hmR0VmxVmcsInfoInit(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
2084	}
2085	}
2086
2087	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2088	/**
2089	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2090	*
2091	* @returns @c true if the MSR is intercepted, @c false otherwise.
2092	* @param pvMsrBitmap The MSR bitmap.
2093	* @param offMsr The MSR byte offset.
2094	* @param iBit The bit offset from the byte offset.
2095	*/
2096	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2097	{
2098	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2099	Assert(pbMsrBitmap);
2100	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2101	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2102	}
2103	#endif
2104
2105	/**
2106	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2107	*
2108	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2109	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2110	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2111	* the read/write access of this MSR.
2112	*
2113	* @param pVCpu The cross context virtual CPU structure.
2114	* @param pVmcsInfo The VMCS info. object.
2115	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2116	* @param idMsr The MSR value.
2117	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2118	* include both a read -and- a write permission!
2119	*
2120	* @sa CPUMGetVmxMsrPermission.
2121	* @remarks Can be called with interrupts disabled.
2122	*/
2123	static void hmR0VmxSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2124	{
2125	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2126	Assert(pbMsrBitmap);
2127	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2128
2129	/*
2130	* MSR-bitmap Layout:
2131	* Byte index MSR range Interpreted as
2132	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2133	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2134	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2135	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2136	*
2137	* A bit corresponding to an MSR within the above range causes a VM-exit
2138	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2139	* the MSR range, it always cause a VM-exit.
2140	*
2141	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2142	*/
2143	uint16_t const offBitmapRead = 0;
2144	uint16_t const offBitmapWrite = 0x800;
2145	uint16_t offMsr;
2146	int32_t iBit;
2147	if (idMsr <= UINT32_C(0x00001fff))
2148	{
2149	offMsr = 0;
2150	iBit = idMsr;
2151	}
2152	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2153	{
2154	offMsr = 0x400;
2155	iBit = idMsr - UINT32_C(0xc0000000);
2156	}
2157	else
2158	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2159
2160	/*
2161	* Set the MSR read permission.
2162	*/
2163	uint16_t const offMsrRead = offBitmapRead + offMsr;
2164	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2165	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2166	{
2167	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2168	bool const fClear = !fIsNstGstVmcs ? true
2169	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2170	#else
2171	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2172	bool const fClear = true;
2173	#endif
2174	if (fClear)
2175	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2176	}
2177	else
2178	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2179
2180	/*
2181	* Set the MSR write permission.
2182	*/
2183	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2184	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2185	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2186	{
2187	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2188	bool const fClear = !fIsNstGstVmcs ? true
2189	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2190	#else
2191	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2192	bool const fClear = true;
2193	#endif
2194	if (fClear)
2195	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2196	}
2197	else
2198	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2199	}
2200
2201
2202	/**
2203	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2204	* area.
2205	*
2206	* @returns VBox status code.
2207	* @param pVCpu The cross context virtual CPU structure.
2208	* @param pVmcsInfo The VMCS info. object.
2209	* @param cMsrs The number of MSRs.
2210	*/
2211	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2212	{
2213	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2214	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2215	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2216	{
2217	/* Commit the MSR counts to the VMCS and update the cache. */
2218	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2219	{
2220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
2222	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2223	pVmcsInfo->cEntryMsrLoad = cMsrs;
2224	pVmcsInfo->cExitMsrStore = cMsrs;
2225	pVmcsInfo->cExitMsrLoad = cMsrs;
2226	}
2227	return VINF_SUCCESS;
2228	}
2229
2230	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2231	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2232	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2233	}
2234
2235
2236	/**
2237	* Adds a new (or updates the value of an existing) guest/host MSR
2238	* pair to be swapped during the world-switch as part of the
2239	* auto-load/store MSR area in the VMCS.
2240	*
2241	* @returns VBox status code.
2242	* @param pVCpu The cross context virtual CPU structure.
2243	* @param pVmxTransient The VMX-transient structure.
2244	* @param idMsr The MSR.
2245	* @param uGuestMsrValue Value of the guest MSR.
2246	* @param fSetReadWrite Whether to set the guest read/write access of this
2247	* MSR (thus not causing a VM-exit).
2248	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2249	* necessary.
2250	*/
2251	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2252	bool fSetReadWrite, bool fUpdateHostMsr)
2253	{
2254	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2255	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2256	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2257	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2258	uint32_t i;
2259
2260	/* Paranoia. */
2261	Assert(pGuestMsrLoad);
2262
2263	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGuestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2264
2265	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2266	for (i = 0; i < cMsrs; i++)
2267	{
2268	if (pGuestMsrLoad[i].u32Msr == idMsr)
2269	break;
2270	}
2271
2272	bool fAdded = false;
2273	if (i == cMsrs)
2274	{
2275	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2276	++cMsrs;
2277	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2278	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2279
2280	/* Set the guest to read/write this MSR without causing VM-exits. */
2281	if ( fSetReadWrite
2282	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2283	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2284
2285	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2286	fAdded = true;
2287	}
2288
2289	/* Update the MSR value for the newly added or already existing MSR. */
2290	pGuestMsrLoad[i].u32Msr = idMsr;
2291	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2292
2293	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2294	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2295	{
2296	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2297	pGuestMsrStore[i].u32Msr = idMsr;
2298	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2299	}
2300
2301	/* Update the corresponding slot in the host MSR area. */
2302	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2303	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2304	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2305	pHostMsr[i].u32Msr = idMsr;
2306
2307	/*
2308	* Only if the caller requests to update the host MSR value AND we've newly added the
2309	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2310	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2311	*
2312	* We do this for performance reasons since reading MSRs may be quite expensive.
2313	*/
2314	if (fAdded)
2315	{
2316	if (fUpdateHostMsr)
2317	{
2318	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2319	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2320	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2321	}
2322	else
2323	{
2324	/* Someone else can do the work. */
2325	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2326	}
2327	}
2328	return VINF_SUCCESS;
2329	}
2330
2331
2332	/**
2333	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2334	* auto-load/store MSR area in the VMCS.
2335	*
2336	* @returns VBox status code.
2337	* @param pVCpu The cross context virtual CPU structure.
2338	* @param pVmxTransient The VMX-transient structure.
2339	* @param idMsr The MSR.
2340	*/
2341	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2342	{
2343	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2344	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2345	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2346	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2347
2348	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2349
2350	for (uint32_t i = 0; i < cMsrs; i++)
2351	{
2352	/* Find the MSR. */
2353	if (pGuestMsrLoad[i].u32Msr == idMsr)
2354	{
2355	/*
2356	* If it's the last MSR, we only need to reduce the MSR count.
2357	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2358	*/
2359	if (i < cMsrs - 1)
2360	{
2361	/* Remove it from the VM-entry MSR-load area. */
2362	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2363	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2364
2365	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2366	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2367	{
2368	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2369	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2370	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2371	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2372	}
2373
2374	/* Remove it from the VM-exit MSR-load area. */
2375	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2376	Assert(pHostMsr[i].u32Msr == idMsr);
2377	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2378	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2379	}
2380
2381	/* Reduce the count to reflect the removed MSR and bail. */
2382	--cMsrs;
2383	break;
2384	}
2385	}
2386
2387	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2388	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2389	{
2390	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2391	AssertRCReturn(rc, rc);
2392
2393	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2394	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2395	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2396
2397	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2398	return VINF_SUCCESS;
2399	}
2400
2401	return VERR_NOT_FOUND;
2402	}
2403
2404
2405	/**
2406	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2407	*
2408	* @returns @c true if found, @c false otherwise.
2409	* @param pVmcsInfo The VMCS info. object.
2410	* @param idMsr The MSR to find.
2411	*/
2412	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2413	{
2414	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2415	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2416	Assert(pMsrs);
2417	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2418	for (uint32_t i = 0; i < cMsrs; i++)
2419	{
2420	if (pMsrs[i].u32Msr == idMsr)
2421	return true;
2422	}
2423	return false;
2424	}
2425
2426
2427	/**
2428	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2429	*
2430	* @param pVCpu The cross context virtual CPU structure.
2431	* @param pVmcsInfo The VMCS info. object.
2432	*
2433	* @remarks No-long-jump zone!!!
2434	*/
2435	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2436	{
2437	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2438
2439	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2440	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2441	Assert(pHostMsrLoad);
2442	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2443	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2444	for (uint32_t i = 0; i < cMsrs; i++)
2445	{
2446	/*
2447	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2448	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2449	*/
2450	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2451	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2452	else
2453	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2454	}
2455	}
2456
2457
2458	/**
2459	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2460	* perform lazy restoration of the host MSRs while leaving VT-x.
2461	*
2462	* @param pVCpu The cross context virtual CPU structure.
2463	*
2464	* @remarks No-long-jump zone!!!
2465	*/
2466	static void hmR0VmxLazySaveHostMsrs(PVMCPUCC pVCpu)
2467	{
2468	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2469
2470	/*
2471	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2472	*/
2473	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2474	{
2475	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2476	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2477	{
2478	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2479	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2480	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2481	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2482	}
2483	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2484	}
2485	}
2486
2487
2488	/**
2489	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2490	* lazily while leaving VT-x.
2491	*
2492	* @returns true if it does, false otherwise.
2493	* @param pVCpu The cross context virtual CPU structure.
2494	* @param idMsr The MSR to check.
2495	*/
2496	static bool hmR0VmxIsLazyGuestMsr(PCVMCPUCC pVCpu, uint32_t idMsr)
2497	{
2498	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2499	{
2500	switch (idMsr)
2501	{
2502	case MSR_K8_LSTAR:
2503	case MSR_K6_STAR:
2504	case MSR_K8_SF_MASK:
2505	case MSR_K8_KERNEL_GS_BASE:
2506	return true;
2507	}
2508	}
2509	return false;
2510	}
2511
2512
2513	/**
2514	* Loads a set of guests MSRs to allow read/passthru to the guest.
2515	*
2516	* The name of this function is slightly confusing. This function does NOT
2517	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2518	* common prefix for functions dealing with "lazy restoration" of the shared
2519	* MSRs.
2520	*
2521	* @param pVCpu The cross context virtual CPU structure.
2522	*
2523	* @remarks No-long-jump zone!!!
2524	*/
2525	static void hmR0VmxLazyLoadGuestMsrs(PVMCPUCC pVCpu)
2526	{
2527	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2528	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2529
2530	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2531	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2532	{
2533	/*
2534	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2535	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2536	* we can skip a few MSR writes.
2537	*
2538	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2539	* guest MSR values in the guest-CPU context might be different to what's currently
2540	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2541	* CPU, see @bugref{8728}.
2542	*/
2543	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2544	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2545	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2546	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2547	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2548	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2549	{
2550	#ifdef VBOX_STRICT
2551	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2552	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2553	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2554	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2555	#endif
2556	}
2557	else
2558	{
2559	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2560	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2561	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2562	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2563	}
2564	}
2565	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2566	}
2567
2568
2569	/**
2570	* Performs lazy restoration of the set of host MSRs if they were previously
2571	* loaded with guest MSR values.
2572	*
2573	* @param pVCpu The cross context virtual CPU structure.
2574	*
2575	* @remarks No-long-jump zone!!!
2576	* @remarks The guest MSRs should have been saved back into the guest-CPU
2577	* context by hmR0VmxImportGuestState()!!!
2578	*/
2579	static void hmR0VmxLazyRestoreHostMsrs(PVMCPUCC pVCpu)
2580	{
2581	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2582	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2583
2584	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2585	{
2586	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2587	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2588	{
2589	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2590	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2591	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2592	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2593	}
2594	}
2595	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2596	}
2597
2598
2599	/**
2600	* Verifies that our cached values of the VMCS fields are all consistent with
2601	* what's actually present in the VMCS.
2602	*
2603	* @returns VBox status code.
2604	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2605	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2606	* VMCS content. HMCPU error-field is
2607	* updated, see VMX_VCI_XXX.
2608	* @param pVCpu The cross context virtual CPU structure.
2609	* @param pVmcsInfo The VMCS info. object.
2610	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2611	*/
2612	static int hmR0VmxCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2613	{
2614	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2615
2616	uint32_t u32Val;
2617	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2618	AssertRC(rc);
2619	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2620	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2621	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2622	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2623
2624	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2625	AssertRC(rc);
2626	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2627	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2628	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2629	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2630
2631	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2632	AssertRC(rc);
2633	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2634	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2635	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2636	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2637
2638	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2639	AssertRC(rc);
2640	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2641	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2642	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2643	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2644
2645	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2646	{
2647	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2648	AssertRC(rc);
2649	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2650	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2651	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2652	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2653	}
2654
2655	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2656	AssertRC(rc);
2657	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2658	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2659	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2660	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2661
2662	uint64_t u64Val;
2663	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2664	AssertRC(rc);
2665	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2666	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2667	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2668	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2669
2670	NOREF(pcszVmcs);
2671	return VINF_SUCCESS;
2672	}
2673
2674
2675	#ifdef VBOX_STRICT
2676	/**
2677	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2678	*
2679	* @param pVCpu The cross context virtual CPU structure.
2680	* @param pVmcsInfo The VMCS info. object.
2681	*/
2682	static void hmR0VmxCheckHostEferMsr(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2683	{
2684	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2685
2686	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2687	{
2688	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2689	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2690	uint64_t uVmcsEferMsrVmcs;
2691	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2692	AssertRC(rc);
2693
2694	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2695	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2696	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2697	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2698	}
2699	}
2700
2701
2702	/**
2703	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2704	* VMCS are correct.
2705	*
2706	* @param pVCpu The cross context virtual CPU structure.
2707	* @param pVmcsInfo The VMCS info. object.
2708	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2709	*/
2710	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2711	{
2712	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2713
2714	/* Read the various MSR-area counts from the VMCS. */
2715	uint32_t cEntryLoadMsrs;
2716	uint32_t cExitStoreMsrs;
2717	uint32_t cExitLoadMsrs;
2718	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2719	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2720	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2721
2722	/* Verify all the MSR counts are the same. */
2723	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2724	Assert(cExitStoreMsrs == cExitLoadMsrs);
2725	uint32_t const cMsrs = cExitLoadMsrs;
2726
2727	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2728	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2729
2730	/* Verify the MSR counts are within the allocated page size. */
2731	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2732
2733	/* Verify the relevant contents of the MSR areas match. */
2734	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2735	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2736	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2737	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2738	for (uint32_t i = 0; i < cMsrs; i++)
2739	{
2740	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2741	if (fSeparateExitMsrStorePage)
2742	{
2743	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2744	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2745	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2746	}
2747
2748	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2749	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2750	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2751
2752	uint64_t const u64HostMsr = ASMRdMsr(pHostMsrLoad->u32Msr);
2753	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64HostMsr,
2754	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2755	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64HostMsr, cMsrs));
2756
2757	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2758	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2759	if (fIsEferMsr)
2760	{
2761	AssertMsgReturnVoid(u64HostMsr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2762	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2763	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64HostMsr, cMsrs));
2764	}
2765
2766	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2767	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2768	{
2769	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2770	if (fIsEferMsr)
2771	{
2772	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2773	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2774	}
2775	else
2776	{
2777	/* Verify LBR MSRs (used only for debugging) are intercepted. We don't passthru these MSRs to the guest yet. */
2778	PCVM pVM = pVCpu->CTX_SUFF(pVM);
2779	if ( pVM->hm.s.vmx.fLbr
2780	&& ( hmR0VmxIsLbrBranchFromMsr(pVM, pGuestMsrLoad->u32Msr, NULL /* pidxMsr */)
2781	\|\| hmR0VmxIsLbrBranchToMsr(pVM, pGuestMsrLoad->u32Msr, NULL /* pidxMsr */)
2782	\|\| pGuestMsrLoad->u32Msr == pVM->hm.s.vmx.idLbrTosMsr))
2783	{
2784	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_MASK) == VMXMSRPM_EXIT_RD_WR,
2785	("u32Msr=%#RX32 cMsrs=%u Passthru read/write for LBR MSRs!\n",
2786	pGuestMsrLoad->u32Msr, cMsrs));
2787	}
2788	else if (!fIsNstGstVmcs)
2789	{
2790	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_MASK) == VMXMSRPM_ALLOW_RD_WR,
2791	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2792	}
2793	else
2794	{
2795	/*
2796	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2797	* execute a nested-guest with MSR passthrough.
2798	*
2799	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2800	* allow passthrough too.
2801	*/
2802	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2803	Assert(pvMsrBitmapNstGst);
2804	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2805	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2806	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2807	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2808	}
2809	}
2810	}
2811
2812	/* Move to the next MSR. */
2813	pHostMsrLoad++;
2814	pGuestMsrLoad++;
2815	pGuestMsrStore++;
2816	}
2817	}
2818	#endif /* VBOX_STRICT */
2819
2820
2821	/**
2822	* Flushes the TLB using EPT.
2823	*
2824	* @returns VBox status code.
2825	* @param pVCpu The cross context virtual CPU structure of the calling
2826	* EMT. Can be NULL depending on @a enmTlbFlush.
2827	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2828	* enmTlbFlush.
2829	* @param enmTlbFlush Type of flush.
2830	*
2831	* @remarks Caller is responsible for making sure this function is called only
2832	* when NestedPaging is supported and providing @a enmTlbFlush that is
2833	* supported by the CPU.
2834	* @remarks Can be called with interrupts disabled.
2835	*/
2836	static void hmR0VmxFlushEpt(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2837	{
2838	uint64_t au64Descriptor[2];
2839	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2840	au64Descriptor[0] = 0;
2841	else
2842	{
2843	Assert(pVCpu);
2844	Assert(pVmcsInfo);
2845	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2846	}
2847	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2848
2849	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2850	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2851
2852	if ( RT_SUCCESS(rc)
2853	&& pVCpu)
2854	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2855	}
2856
2857
2858	/**
2859	* Flushes the TLB using VPID.
2860	*
2861	* @returns VBox status code.
2862	* @param pVCpu The cross context virtual CPU structure of the calling
2863	* EMT. Can be NULL depending on @a enmTlbFlush.
2864	* @param enmTlbFlush Type of flush.
2865	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2866	* on @a enmTlbFlush).
2867	*
2868	* @remarks Can be called with interrupts disabled.
2869	*/
2870	static void hmR0VmxFlushVpid(PVMCPUCC pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2871	{
2872	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2873
2874	uint64_t au64Descriptor[2];
2875	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2876	{
2877	au64Descriptor[0] = 0;
2878	au64Descriptor[1] = 0;
2879	}
2880	else
2881	{
2882	AssertPtr(pVCpu);
2883	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2884	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2885	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2886	au64Descriptor[1] = GCPtr;
2887	}
2888
2889	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2890	AssertMsg(rc == VINF_SUCCESS,
2891	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2892
2893	if ( RT_SUCCESS(rc)
2894	&& pVCpu)
2895	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2896	NOREF(rc);
2897	}
2898
2899
2900	/**
2901	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2902	* otherwise there is nothing really to invalidate.
2903	*
2904	* @returns VBox status code.
2905	* @param pVCpu The cross context virtual CPU structure.
2906	* @param GCVirt Guest virtual address of the page to invalidate.
2907	*/
2908	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
2909	{
2910	AssertPtr(pVCpu);
2911	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2912
2913	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2914	{
2915	/*
2916	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2917	* the EPT case. See @bugref{6043} and @bugref{6177}.
2918	*
2919	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2920	* as this function maybe called in a loop with individual addresses.
2921	*/
2922	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2923	if (pVM->hm.s.vmx.fVpid)
2924	{
2925	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2926	if (fVpidFlush)
2927	{
2928	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2929	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2930	}
2931	else
2932	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2933	}
2934	else if (pVM->hm.s.fNestedPaging)
2935	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2936	}
2937
2938	return VINF_SUCCESS;
2939	}
2940
2941
2942	/**
2943	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2944	* case where neither EPT nor VPID is supported by the CPU.
2945	*
2946	* @param pHostCpu The HM physical-CPU structure.
2947	* @param pVCpu The cross context virtual CPU structure.
2948	*
2949	* @remarks Called with interrupts disabled.
2950	*/
2951	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2952	{
2953	AssertPtr(pVCpu);
2954	AssertPtr(pHostCpu);
2955
2956	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2957
2958	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2959	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2960	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2961	pVCpu->hm.s.fForceTLBFlush = false;
2962	return;
2963	}
2964
2965
2966	/**
2967	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2968	*
2969	* @param pHostCpu The HM physical-CPU structure.
2970	* @param pVCpu The cross context virtual CPU structure.
2971	* @param pVmcsInfo The VMCS info. object.
2972	*
2973	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2974	* nomenclature. The reason is, to avoid confusion in compare statements
2975	* since the host-CPU copies are named "ASID".
2976	*
2977	* @remarks Called with interrupts disabled.
2978	*/
2979	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2980	{
2981	#ifdef VBOX_WITH_STATISTICS
2982	bool fTlbFlushed = false;
2983	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2984	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2985	if (!fTlbFlushed) \
2986	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2987	} while (0)
2988	#else
2989	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2990	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2991	#endif
2992
2993	AssertPtr(pVCpu);
2994	AssertPtr(pHostCpu);
2995	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2996
2997	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2998	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2999	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
3000	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
3001
3002	/*
3003	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
3004	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3005	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3006	* cannot reuse the current ASID anymore.
3007	*/
3008	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3009	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3010	{
3011	++pHostCpu->uCurrentAsid;
3012	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3013	{
3014	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
3015	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3016	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3017	}
3018
3019	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3020	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3021	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3022
3023	/*
3024	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
3025	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
3026	*/
3027	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3028	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3029	HMVMX_SET_TAGGED_TLB_FLUSHED();
3030	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
3031	}
3032	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
3033	{
3034	/*
3035	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
3036	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
3037	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
3038	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
3039	* mappings, see @bugref{6568}.
3040	*
3041	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
3042	*/
3043	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3044	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3045	HMVMX_SET_TAGGED_TLB_FLUSHED();
3046	}
3047	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3048	{
3049	/*
3050	* The nested-guest specifies its own guest-physical address to use as the APIC-access
3051	* address which requires flushing the TLB of EPT cached structures.
3052	*
3053	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
3054	*/
3055	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3056	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3057	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3058	HMVMX_SET_TAGGED_TLB_FLUSHED();
3059	}
3060
3061
3062	pVCpu->hm.s.fForceTLBFlush = false;
3063	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3064
3065	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3066	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3067	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3068	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3069	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3070	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3071	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3072	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3073	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3074
3075	/* Update VMCS with the VPID. */
3076	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3077	AssertRC(rc);
3078
3079	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3080	}
3081
3082
3083	/**
3084	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3085	*
3086	* @param pHostCpu The HM physical-CPU structure.
3087	* @param pVCpu The cross context virtual CPU structure.
3088	* @param pVmcsInfo The VMCS info. object.
3089	*
3090	* @remarks Called with interrupts disabled.
3091	*/
3092	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3093	{
3094	AssertPtr(pVCpu);
3095	AssertPtr(pHostCpu);
3096	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3097	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3098	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3099
3100	/*
3101	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3102	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3103	*/
3104	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3105	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3106	{
3107	pVCpu->hm.s.fForceTLBFlush = true;
3108	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3109	}
3110
3111	/* Check for explicit TLB flushes. */
3112	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3113	{
3114	pVCpu->hm.s.fForceTLBFlush = true;
3115	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3116	}
3117
3118	/* Check for TLB flushes while switching to/from a nested-guest. */
3119	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3120	{
3121	pVCpu->hm.s.fForceTLBFlush = true;
3122	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3123	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3124	}
3125
3126	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3127	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3128
3129	if (pVCpu->hm.s.fForceTLBFlush)
3130	{
3131	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3132	pVCpu->hm.s.fForceTLBFlush = false;
3133	}
3134	}
3135
3136
3137	/**
3138	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3139	*
3140	* @param pHostCpu The HM physical-CPU structure.
3141	* @param pVCpu The cross context virtual CPU structure.
3142	*
3143	* @remarks Called with interrupts disabled.
3144	*/
3145	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
3146	{
3147	AssertPtr(pVCpu);
3148	AssertPtr(pHostCpu);
3149	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3150	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3151	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3152
3153	/*
3154	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3155	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3156	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3157	* cannot reuse the current ASID anymore.
3158	*/
3159	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3160	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3161	{
3162	pVCpu->hm.s.fForceTLBFlush = true;
3163	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3164	}
3165
3166	/* Check for explicit TLB flushes. */
3167	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3168	{
3169	/*
3170	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3171	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3172	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3173	* include fExplicitFlush's too) - an obscure corner case.
3174	*/
3175	pVCpu->hm.s.fForceTLBFlush = true;
3176	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3177	}
3178
3179	/* Check for TLB flushes while switching to/from a nested-guest. */
3180	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3181	{
3182	pVCpu->hm.s.fForceTLBFlush = true;
3183	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3184	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3185	}
3186
3187	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3188	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3189	if (pVCpu->hm.s.fForceTLBFlush)
3190	{
3191	++pHostCpu->uCurrentAsid;
3192	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3193	{
3194	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3195	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3196	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3197	}
3198
3199	pVCpu->hm.s.fForceTLBFlush = false;
3200	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3201	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3202	if (pHostCpu->fFlushAsidBeforeUse)
3203	{
3204	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3205	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3206	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3207	{
3208	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3209	pHostCpu->fFlushAsidBeforeUse = false;
3210	}
3211	else
3212	{
3213	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3214	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3215	}
3216	}
3217	}
3218
3219	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3220	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3221	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3222	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3223	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3224	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3225	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3226
3227	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3228	AssertRC(rc);
3229	}
3230
3231
3232	/**
3233	* Flushes the guest TLB entry based on CPU capabilities.
3234	*
3235	* @param pHostCpu The HM physical-CPU structure.
3236	* @param pVCpu The cross context virtual CPU structure.
3237	* @param pVmcsInfo The VMCS info. object.
3238	*
3239	* @remarks Called with interrupts disabled.
3240	*/
3241	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3242	{
3243	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3244	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3245	#endif
3246	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3247	switch (pVM->hm.s.vmx.enmTlbFlushType)
3248	{
3249	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3250	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3251	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3252	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3253	default:
3254	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3255	break;
3256	}
3257	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3258	}
3259
3260
3261	/**
3262	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3263	* TLB entries from the host TLB before VM-entry.
3264	*
3265	* @returns VBox status code.
3266	* @param pVM The cross context VM structure.
3267	*/
3268	static int hmR0VmxSetupTaggedTlb(PVMCC pVM)
3269	{
3270	/*
3271	* Determine optimal flush type for nested paging.
3272	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3273	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3274	*/
3275	if (pVM->hm.s.fNestedPaging)
3276	{
3277	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3278	{
3279	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3280	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3281	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3282	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3283	else
3284	{
3285	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3286	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3287	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3288	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3289	}
3290
3291	/* Make sure the write-back cacheable memory type for EPT is supported. */
3292	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3293	{
3294	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3295	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3296	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3297	}
3298
3299	/* EPT requires a page-walk length of 4. */
3300	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3301	{
3302	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3303	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3304	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3305	}
3306	}
3307	else
3308	{
3309	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3310	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3311	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3312	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3313	}
3314	}
3315
3316	/*
3317	* Determine optimal flush type for VPID.
3318	*/
3319	if (pVM->hm.s.vmx.fVpid)
3320	{
3321	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3322	{
3323	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3324	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3325	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3326	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3327	else
3328	{
3329	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3330	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3331	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3332	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3333	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3334	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3335	pVM->hm.s.vmx.fVpid = false;
3336	}
3337	}
3338	else
3339	{
3340	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3341	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3342	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3343	pVM->hm.s.vmx.fVpid = false;
3344	}
3345	}
3346
3347	/*
3348	* Setup the handler for flushing tagged-TLBs.
3349	*/
3350	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3351	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3352	else if (pVM->hm.s.fNestedPaging)
3353	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3354	else if (pVM->hm.s.vmx.fVpid)
3355	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3356	else
3357	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3358	return VINF_SUCCESS;
3359	}
3360
3361
3362	/**
3363	* Sets up the LBR MSR ranges based on the host CPU.
3364	*
3365	* @returns VBox status code.
3366	* @param pVM The cross context VM structure.
3367	*/
3368	static int hmR0VmxSetupLbrMsrRange(PVMCC pVM)
3369	{
3370	Assert(pVM->hm.s.vmx.fLbr);
3371	uint32_t idLbrFromIpMsrFirst;
3372	uint32_t idLbrFromIpMsrLast;
3373	uint32_t idLbrToIpMsrFirst;
3374	uint32_t idLbrToIpMsrLast;
3375	uint32_t idLbrTosMsr;
3376
3377	/*
3378	* Determine the LBR MSRs supported for this host CPU family and model.
3379	*
3380	* See Intel spec. 17.4.8 "LBR Stack".
3381	* See Intel "Model-Specific Registers" spec.
3382	*/
3383	uint32_t const uFamilyModel = (pVM->cpum.ro.HostFeatures.uFamily << 8)
3384	\| pVM->cpum.ro.HostFeatures.uModel;
3385	switch (uFamilyModel)
3386	{
3387	case 0x0f01: case 0x0f02:
3388	idLbrFromIpMsrFirst = MSR_P4_LASTBRANCH_0;
3389	idLbrFromIpMsrLast = MSR_P4_LASTBRANCH_3;
3390	idLbrToIpMsrFirst = 0x0;
3391	idLbrToIpMsrLast = 0x0;
3392	idLbrTosMsr = MSR_P4_LASTBRANCH_TOS;
3393	break;
3394
3395	case 0x065c: case 0x065f: case 0x064e: case 0x065e: case 0x068e:
3396	case 0x069e: case 0x0655: case 0x0666: case 0x067a: case 0x0667:
3397	case 0x066a: case 0x066c: case 0x067d: case 0x067e:
3398	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
3399	idLbrFromIpMsrLast = MSR_LASTBRANCH_31_FROM_IP;
3400	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
3401	idLbrToIpMsrLast = MSR_LASTBRANCH_31_TO_IP;
3402	idLbrTosMsr = MSR_LASTBRANCH_TOS;
3403	break;
3404
3405	case 0x063d: case 0x0647: case 0x064f: case 0x0656: case 0x063c:
3406	case 0x0645: case 0x0646: case 0x063f: case 0x062a: case 0x062d:
3407	case 0x063a: case 0x063e: case 0x061a: case 0x061e: case 0x061f:
3408	case 0x062e: case 0x0625: case 0x062c: case 0x062f:
3409	idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
3410	idLbrFromIpMsrLast = MSR_LASTBRANCH_15_FROM_IP;
3411	idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
3412	idLbrToIpMsrLast = MSR_LASTBRANCH_15_TO_IP;
3413	idLbrTosMsr = MSR_LASTBRANCH_TOS;
3414	break;
3415
3416	case 0x0617: case 0x061d: case 0x060f:
3417	idLbrFromIpMsrFirst = MSR_CORE2_LASTBRANCH_0_FROM_IP;
3418	idLbrFromIpMsrLast = MSR_CORE2_LASTBRANCH_3_FROM_IP;
3419	idLbrToIpMsrFirst = MSR_CORE2_LASTBRANCH_0_TO_IP;
3420	idLbrToIpMsrLast = MSR_CORE2_LASTBRANCH_3_TO_IP;
3421	idLbrTosMsr = MSR_CORE2_LASTBRANCH_TOS;
3422	break;
3423
3424	/* Atom and related microarchitectures we don't care about:
3425	case 0x0637: case 0x064a: case 0x064c: case 0x064d: case 0x065a:
3426	case 0x065d: case 0x061c: case 0x0626: case 0x0627: case 0x0635:
3427	case 0x0636: */
3428	/* All other CPUs: */
3429	default:
3430	{
3431	LogRelFunc(("Could not determine LBR stack size for the CPU model %#x\n", uFamilyModel));
3432	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_UNKNOWN;
3433	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3434	}
3435	}
3436
3437	/*
3438	* Validate.
3439	*/
3440	uint32_t const cLbrStack = idLbrFromIpMsrLast - idLbrFromIpMsrFirst + 1;
3441	PCVMCPU pVCpu0 = VMCC_GET_CPU_0(pVM);
3442	AssertCompile( RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr)
3443	== RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrToIpMsr));
3444	if (cLbrStack > RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr))
3445	{
3446	LogRelFunc(("LBR stack size of the CPU (%u) exceeds our buffer size\n", cLbrStack));
3447	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_OVERFLOW;
3448	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3449	}
3450	NOREF(pVCpu0);
3451
3452	/*
3453	* Update the LBR info. to the VM struct. for use later.
3454	*/
3455	pVM->hm.s.vmx.idLbrTosMsr = idLbrTosMsr;
3456	pVM->hm.s.vmx.idLbrFromIpMsrFirst = idLbrFromIpMsrFirst;
3457	pVM->hm.s.vmx.idLbrFromIpMsrLast = idLbrFromIpMsrLast;
3458
3459	pVM->hm.s.vmx.idLbrToIpMsrFirst = idLbrToIpMsrFirst;
3460	pVM->hm.s.vmx.idLbrToIpMsrLast = idLbrToIpMsrLast;
3461	return VINF_SUCCESS;
3462	}
3463
3464
3465	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3466	/**
3467	* Sets up the shadow VMCS fields arrays.
3468	*
3469	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3470	* executing the guest.
3471	*
3472	* @returns VBox status code.
3473	* @param pVM The cross context VM structure.
3474	*/
3475	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVMCC pVM)
3476	{
3477	/*
3478	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3479	* when the host does not support it.
3480	*/
3481	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3482	if ( !fGstVmwriteAll
3483	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3484	{ /* likely. */ }
3485	else
3486	{
3487	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3488	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3489	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3490	}
3491
3492	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3493	uint32_t cRwFields = 0;
3494	uint32_t cRoFields = 0;
3495	for (uint32_t i = 0; i < cVmcsFields; i++)
3496	{
3497	VMXVMCSFIELD VmcsField;
3498	VmcsField.u = g_aVmcsFields[i];
3499
3500	/*
3501	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3502	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3503	* in the shadow VMCS fields array as they would be redundant.
3504	*
3505	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3506	* we must not include it in the shadow VMCS fields array. Guests attempting to
3507	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3508	* the required behavior.
3509	*/
3510	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3511	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3512	{
3513	/*
3514	* Read-only fields are placed in a separate array so that while syncing shadow
3515	* VMCS fields later (which is more performance critical) we can avoid branches.
3516	*
3517	* However, if the guest can write to all fields (including read-only fields),
3518	* we treat it a as read/write field. Otherwise, writing to these fields would
3519	* cause a VMWRITE instruction error while syncing the shadow VMCS.
3520	*/
3521	if ( fGstVmwriteAll
3522	\|\| !VMXIsVmcsFieldReadOnly(VmcsField.u))
3523	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3524	else
3525	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3526	}
3527	}
3528
3529	/* Update the counts. */
3530	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3531	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3532	return VINF_SUCCESS;
3533	}
3534
3535
3536	/**
3537	* Sets up the VMREAD and VMWRITE bitmaps.
3538	*
3539	* @param pVM The cross context VM structure.
3540	*/
3541	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVMCC pVM)
3542	{
3543	/*
3544	* By default, ensure guest attempts to access any VMCS fields cause VM-exits.
3545	*/
3546	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3547	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3548	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3549	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3550	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3551
3552	/*
3553	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3554	* VMREAD and VMWRITE bitmaps.
3555	*/
3556	{
3557	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3558	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3559	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3560	{
3561	uint32_t const uVmcsField = paShadowVmcsFields[i];
3562	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3563	Assert(uVmcsField >> 3 < cbBitmap);
3564	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3565	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3566	}
3567	}
3568
3569	/*
3570	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3571	* if the host supports VMWRITE to all supported VMCS fields.
3572	*/
3573	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3574	{
3575	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3576	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3577	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3578	{
3579	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3580	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3581	Assert(uVmcsField >> 3 < cbBitmap);
3582	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3583	}
3584	}
3585	}
3586	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3587
3588
3589	/**
3590	* Sets up the virtual-APIC page address for the VMCS.
3591	*
3592	* @param pVmcsInfo The VMCS info. object.
3593	*/
3594	DECLINLINE(void) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
3595	{
3596	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3597	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3598	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3599	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3600	AssertRC(rc);
3601	}
3602
3603
3604	/**
3605	* Sets up the MSR-bitmap address for the VMCS.
3606	*
3607	* @param pVmcsInfo The VMCS info. object.
3608	*/
3609	DECLINLINE(void) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
3610	{
3611	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3612	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3613	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3614	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3615	AssertRC(rc);
3616	}
3617
3618
3619	/**
3620	* Sets up the APIC-access page address for the VMCS.
3621	*
3622	* @param pVCpu The cross context virtual CPU structure.
3623	*/
3624	DECLINLINE(void) hmR0VmxSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
3625	{
3626	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3627	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3628	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3629	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3630	AssertRC(rc);
3631	}
3632
3633
3634	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3635	/**
3636	* Sets up the VMREAD bitmap address for the VMCS.
3637	*
3638	* @param pVCpu The cross context virtual CPU structure.
3639	*/
3640	DECLINLINE(void) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
3641	{
3642	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3643	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3644	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3645	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3646	AssertRC(rc);
3647	}
3648
3649
3650	/**
3651	* Sets up the VMWRITE bitmap address for the VMCS.
3652	*
3653	* @param pVCpu The cross context virtual CPU structure.
3654	*/
3655	DECLINLINE(void) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
3656	{
3657	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3658	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3659	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3660	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3661	AssertRC(rc);
3662	}
3663	#endif
3664
3665
3666	/**
3667	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3668	* in the VMCS.
3669	*
3670	* @returns VBox status code.
3671	* @param pVmcsInfo The VMCS info. object.
3672	*/
3673	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
3674	{
3675	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3676	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3677	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3678
3679	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3680	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3681	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3682
3683	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3684	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3685	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3686
3687	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad); AssertRC(rc);
3688	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore); AssertRC(rc);
3689	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad); AssertRC(rc);
3690	return VINF_SUCCESS;
3691	}
3692
3693
3694	/**
3695	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3696	*
3697	* @param pVCpu The cross context virtual CPU structure.
3698	* @param pVmcsInfo The VMCS info. object.
3699	*/
3700	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3701	{
3702	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3703
3704	/*
3705	* By default, ensure guest attempts to access any MSR cause VM-exits.
3706	* This shall later be relaxed for specific MSRs as necessary.
3707	*
3708	* Note: For nested-guests, the entire bitmap will be merged prior to
3709	* executing the nested-guest using hardware-assisted VMX and hence there
3710	* is no need to perform this operation. See hmR0VmxMergeMsrBitmapNested.
3711	*/
3712	Assert(pVmcsInfo->pvMsrBitmap);
3713	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
3714
3715	/*
3716	* The guest can access the following MSRs (read, write) without causing
3717	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3718	*/
3719	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3720	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3721	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3722	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3723	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3724	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3725
3726	/*
3727	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3728	* associated with then. We never need to intercept access (writes need to be
3729	* executed without causing a VM-exit, reads will #GP fault anyway).
3730	*
3731	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3732	* read/write them. We swap the the guest/host MSR value using the
3733	* auto-load/store MSR area.
3734	*/
3735	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3736	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3737	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3738	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3739	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3740	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3741
3742	/*
3743	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3744	* required for 64-bit guests.
3745	*/
3746	if (pVM->hm.s.fAllow64BitGuests)
3747	{
3748	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3749	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3750	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3751	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3752	}
3753
3754	/*
3755	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3756	*/
3757	#ifdef VBOX_STRICT
3758	Assert(pVmcsInfo->pvMsrBitmap);
3759	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3760	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3761	#endif
3762	}
3763
3764
3765	/**
3766	* Sets up pin-based VM-execution controls in the VMCS.
3767	*
3768	* @returns VBox status code.
3769	* @param pVCpu The cross context virtual CPU structure.
3770	* @param pVmcsInfo The VMCS info. object.
3771	*/
3772	static int hmR0VmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3773	{
3774	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3775	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3776	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3777
3778	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3779	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3780
3781	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3782	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3783
3784	/* Enable the VMX-preemption timer. */
3785	if (pVM->hm.s.vmx.fUsePreemptTimer)
3786	{
3787	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3788	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3789	}
3790
3791	#if 0
3792	/* Enable posted-interrupt processing. */
3793	if (pVM->hm.s.fPostedIntrs)
3794	{
3795	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3796	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3797	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3798	}
3799	#endif
3800
3801	if ((fVal & fZap) != fVal)
3802	{
3803	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3804	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3805	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3806	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3807	}
3808
3809	/* Commit it to the VMCS and update our cache. */
3810	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3811	AssertRC(rc);
3812	pVmcsInfo->u32PinCtls = fVal;
3813
3814	return VINF_SUCCESS;
3815	}
3816
3817
3818	/**
3819	* Sets up secondary processor-based VM-execution controls in the VMCS.
3820	*
3821	* @returns VBox status code.
3822	* @param pVCpu The cross context virtual CPU structure.
3823	* @param pVmcsInfo The VMCS info. object.
3824	*/
3825	static int hmR0VmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3826	{
3827	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3828	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3829	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3830
3831	/* WBINVD causes a VM-exit. */
3832	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3833	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3834
3835	/* Enable EPT (aka nested-paging). */
3836	if (pVM->hm.s.fNestedPaging)
3837	fVal \|= VMX_PROC_CTLS2_EPT;
3838
3839	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3840	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3841	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3842	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3843	fVal \|= VMX_PROC_CTLS2_INVPCID;
3844
3845	/* Enable VPID. */
3846	if (pVM->hm.s.vmx.fVpid)
3847	fVal \|= VMX_PROC_CTLS2_VPID;
3848
3849	/* Enable unrestricted guest execution. */
3850	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3851	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3852
3853	#if 0
3854	if (pVM->hm.s.fVirtApicRegs)
3855	{
3856	/* Enable APIC-register virtualization. */
3857	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3858	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3859
3860	/* Enable virtual-interrupt delivery. */
3861	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3862	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3863	}
3864	#endif
3865
3866	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3867	where the TPR shadow resides. */
3868	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3869	* done dynamically. */
3870	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3871	{
3872	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3873	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3874	}
3875
3876	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3877	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3878	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3879	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3880	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3881
3882	/* Enable Pause-Loop exiting. */
3883	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3884	&& pVM->hm.s.vmx.cPleGapTicks
3885	&& pVM->hm.s.vmx.cPleWindowTicks)
3886	{
3887	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3888
3889	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
3890	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
3891	}
3892
3893	if ((fVal & fZap) != fVal)
3894	{
3895	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3896	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3897	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3898	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3899	}
3900
3901	/* Commit it to the VMCS and update our cache. */
3902	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3903	AssertRC(rc);
3904	pVmcsInfo->u32ProcCtls2 = fVal;
3905
3906	return VINF_SUCCESS;
3907	}
3908
3909
3910	/**
3911	* Sets up processor-based VM-execution controls in the VMCS.
3912	*
3913	* @returns VBox status code.
3914	* @param pVCpu The cross context virtual CPU structure.
3915	* @param pVmcsInfo The VMCS info. object.
3916	*/
3917	static int hmR0VmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3918	{
3919	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3920	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3921	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3922
3923	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3924	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3925	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3926	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3927	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3928	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3929	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3930
3931	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3932	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3933	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3934	{
3935	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3936	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3937	}
3938
3939	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3940	if (!pVM->hm.s.fNestedPaging)
3941	{
3942	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3943	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3944	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3945	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3946	}
3947
3948	/* Use TPR shadowing if supported by the CPU. */
3949	if ( PDMHasApic(pVM)
3950	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
3951	{
3952	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3953	/* CR8 writes cause a VM-exit based on TPR threshold. */
3954	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3955	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3956	hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3957	}
3958	else
3959	{
3960	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3961	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3962	if (pVM->hm.s.fAllow64BitGuests)
3963	{
3964	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3965	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3966	}
3967	}
3968
3969	/* Use MSR-bitmaps if supported by the CPU. */
3970	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3971	{
3972	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3973	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3974	}
3975
3976	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3977	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3978	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3979
3980	if ((fVal & fZap) != fVal)
3981	{
3982	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3983	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3984	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3985	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3986	}
3987
3988	/* Commit it to the VMCS and update our cache. */
3989	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3990	AssertRC(rc);
3991	pVmcsInfo->u32ProcCtls = fVal;
3992
3993	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3994	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3995	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3996
3997	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3998	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3999	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
4000
4001	/* Sanity check, should not really happen. */
4002	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
4003	{ /* likely */ }
4004	else
4005	{
4006	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
4007	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4008	}
4009
4010	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
4011	return VINF_SUCCESS;
4012	}
4013
4014
4015	/**
4016	* Sets up miscellaneous (everything other than Pin, Processor and secondary
4017	* Processor-based VM-execution) control fields in the VMCS.
4018	*
4019	* @returns VBox status code.
4020	* @param pVCpu The cross context virtual CPU structure.
4021	* @param pVmcsInfo The VMCS info. object.
4022	*/
4023	static int hmR0VmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4024	{
4025	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4026	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
4027	{
4028	hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
4029	hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
4030	}
4031	#endif
4032
4033	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
4034	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
4035	AssertRC(rc);
4036
4037	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
4038	if (RT_SUCCESS(rc))
4039	{
4040	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
4041	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
4042
4043	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
4044	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
4045
4046	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
4047	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
4048
4049	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fLbr)
4050	{
4051	rc = VMXWriteVmcsNw(VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
4052	AssertRC(rc);
4053	}
4054	return VINF_SUCCESS;
4055	}
4056	else
4057	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
4058	return rc;
4059	}
4060
4061
4062	/**
4063	* Sets up the initial exception bitmap in the VMCS based on static conditions.
4064	*
4065	* We shall setup those exception intercepts that don't change during the
4066	* lifetime of the VM here. The rest are done dynamically while loading the
4067	* guest state.
4068	*
4069	* @param pVCpu The cross context virtual CPU structure.
4070	* @param pVmcsInfo The VMCS info. object.
4071	*/
4072	static void hmR0VmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4073	{
4074	/*
4075	* The following exceptions are always intercepted:
4076	*
4077	* #AC - To prevent the guest from hanging the CPU.
4078	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
4079	* recursive #DBs can cause a CPU hang.
4080	* #PF - To sync our shadow page tables when nested-paging is not used.
4081	*/
4082	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
4083	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
4084	\| RT_BIT(X86_XCPT_DB)
4085	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
4086
4087	/* Commit it to the VMCS. */
4088	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4089	AssertRC(rc);
4090
4091	/* Update our cache of the exception bitmap. */
4092	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4093	}
4094
4095
4096	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4097	/**
4098	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
4099	*
4100	* @returns VBox status code.
4101	* @param pVCpu The cross context virtual CPU structure.
4102	* @param pVmcsInfo The VMCS info. object.
4103	*/
4104	static int hmR0VmxSetupVmcsCtlsNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4105	{
4106	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
4107	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
4108	AssertRC(rc);
4109
4110	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
4111	if (RT_SUCCESS(rc))
4112	{
4113	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4114	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
4115
4116	/* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
4117	Assert(!pVmcsInfo->u64Cr0Mask);
4118	Assert(!pVmcsInfo->u64Cr4Mask);
4119	return VINF_SUCCESS;
4120	}
4121	else
4122	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
4123	return rc;
4124	}
4125	#endif
4126
4127
4128	/**
4129	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
4130	* VMX.
4131	*
4132	* @returns VBox status code.
4133	* @param pVCpu The cross context virtual CPU structure.
4134	* @param pVmcsInfo The VMCS info. object.
4135	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
4136	*/
4137	static int hmR0VmxSetupVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
4138	{
4139	Assert(pVmcsInfo->pvVmcs);
4140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4141
4142	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
4143	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4144	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4145	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
4146
4147	LogFlowFunc(("\n"));
4148
4149	/*
4150	* Initialize the VMCS using VMCLEAR before loading the VMCS.
4151	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
4152	*/
4153	int rc = hmR0VmxClearVmcs(pVmcsInfo);
4154	if (RT_SUCCESS(rc))
4155	{
4156	rc = hmR0VmxLoadVmcs(pVmcsInfo);
4157	if (RT_SUCCESS(rc))
4158	{
4159	/*
4160	* Initialize the hardware-assisted VMX execution handler for guest and nested-guest VMCS.
4161	* The host is always 64-bit since we no longer support 32-bit hosts.
4162	* Currently we have just a single handler for all guest modes as well, see @bugref{6208#c73}.
4163	*/
4164	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
4165	if (!fIsNstGstVmcs)
4166	{
4167	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
4168	if (RT_SUCCESS(rc))
4169	{
4170	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4171	if (RT_SUCCESS(rc))
4172	{
4173	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4174	if (RT_SUCCESS(rc))
4175	{
4176	hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4177	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4178	/*
4179	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4180	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4181	* making it fit for use when VMCS shadowing is later enabled.
4182	*/
4183	if (pVmcsInfo->pvShadowVmcs)
4184	{
4185	VMXVMCSREVID VmcsRevId;
4186	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4187	VmcsRevId.n.fIsShadowVmcs = 1;
4188	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4189	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4190	if (RT_SUCCESS(rc))
4191	{ /* likely */ }
4192	else
4193	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4194	}
4195	#endif
4196	}
4197	else
4198	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4199	}
4200	else
4201	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4202	}
4203	else
4204	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4205	}
4206	else
4207	{
4208	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4209	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4210	if (RT_SUCCESS(rc))
4211	{ /* likely */ }
4212	else
4213	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4214	#else
4215	AssertFailed();
4216	#endif
4217	}
4218	}
4219	else
4220	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4221	}
4222	else
4223	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4224
4225	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4226	if (RT_SUCCESS(rc))
4227	{
4228	rc = hmR0VmxClearVmcs(pVmcsInfo);
4229	if (RT_SUCCESS(rc))
4230	{ /* likely */ }
4231	else
4232	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4233	}
4234
4235	/*
4236	* Update the last-error record both for failures and success, so we
4237	* can propagate the status code back to ring-3 for diagnostics.
4238	*/
4239	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4240	NOREF(pszVmcs);
4241	return rc;
4242	}
4243
4244
4245	/**
4246	* Does global VT-x initialization (called during module initialization).
4247	*
4248	* @returns VBox status code.
4249	*/
4250	VMMR0DECL(int) VMXR0GlobalInit(void)
4251	{
4252	#ifdef HMVMX_USE_FUNCTION_TABLE
4253	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_aVMExitHandlers));
4254	# ifdef VBOX_STRICT
4255	for (unsigned i = 0; i < RT_ELEMENTS(g_aVMExitHandlers); i++)
4256	Assert(g_aVMExitHandlers[i].pfn);
4257	# endif
4258	#endif
4259	return VINF_SUCCESS;
4260	}
4261
4262
4263	/**
4264	* Does global VT-x termination (called during module termination).
4265	*/
4266	VMMR0DECL(void) VMXR0GlobalTerm()
4267	{
4268	/* Nothing to do currently. */
4269	}
4270
4271
4272	/**
4273	* Sets up and activates VT-x on the current CPU.
4274	*
4275	* @returns VBox status code.
4276	* @param pHostCpu The HM physical-CPU structure.
4277	* @param pVM The cross context VM structure. Can be
4278	* NULL after a host resume operation.
4279	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4280	* fEnabledByHost is @c true).
4281	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4282	* @a fEnabledByHost is @c true).
4283	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4284	* enable VT-x on the host.
4285	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4286	*/
4287	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4288	PCSUPHWVIRTMSRS pHwvirtMsrs)
4289	{
4290	AssertPtr(pHostCpu);
4291	AssertPtr(pHwvirtMsrs);
4292	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4293
4294	/* Enable VT-x if it's not already enabled by the host. */
4295	if (!fEnabledByHost)
4296	{
4297	int rc = hmR0VmxEnterRootMode(pHostCpu, pVM, HCPhysCpuPage, pvCpuPage);
4298	if (RT_FAILURE(rc))
4299	return rc;
4300	}
4301
4302	/*
4303	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4304	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4305	* invalidated when flushing by VPID.
4306	*/
4307	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4308	{
4309	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4310	pHostCpu->fFlushAsidBeforeUse = false;
4311	}
4312	else
4313	pHostCpu->fFlushAsidBeforeUse = true;
4314
4315	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4316	++pHostCpu->cTlbFlushes;
4317
4318	return VINF_SUCCESS;
4319	}
4320
4321
4322	/**
4323	* Deactivates VT-x on the current CPU.
4324	*
4325	* @returns VBox status code.
4326	* @param pHostCpu The HM physical-CPU structure.
4327	* @param pvCpuPage Pointer to the VMXON region.
4328	* @param HCPhysCpuPage Physical address of the VMXON region.
4329	*
4330	* @remarks This function should never be called when SUPR0EnableVTx() or
4331	* similar was used to enable VT-x on the host.
4332	*/
4333	VMMR0DECL(int) VMXR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4334	{
4335	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4336
4337	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4338	return hmR0VmxLeaveRootMode(pHostCpu);
4339	}
4340
4341
4342	/**
4343	* Does per-VM VT-x initialization.
4344	*
4345	* @returns VBox status code.
4346	* @param pVM The cross context VM structure.
4347	*/
4348	VMMR0DECL(int) VMXR0InitVM(PVMCC pVM)
4349	{
4350	AssertPtr(pVM);
4351	LogFlowFunc(("pVM=%p\n", pVM));
4352
4353	hmR0VmxStructsInit(pVM);
4354	int rc = hmR0VmxStructsAlloc(pVM);
4355	if (RT_FAILURE(rc))
4356	{
4357	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4358	return rc;
4359	}
4360
4361	/* Setup the crash dump page. */
4362	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4363	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
4364	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
4365	#endif
4366	return VINF_SUCCESS;
4367	}
4368
4369
4370	/**
4371	* Does per-VM VT-x termination.
4372	*
4373	* @returns VBox status code.
4374	* @param pVM The cross context VM structure.
4375	*/
4376	VMMR0DECL(int) VMXR0TermVM(PVMCC pVM)
4377	{
4378	AssertPtr(pVM);
4379	LogFlowFunc(("pVM=%p\n", pVM));
4380
4381	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4382	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4383	{
4384	Assert(pVM->hm.s.vmx.pvScratch);
4385	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4386	}
4387	#endif
4388	hmR0VmxStructsFree(pVM);
4389	return VINF_SUCCESS;
4390	}
4391
4392
4393	/**
4394	* Sets up the VM for execution using hardware-assisted VMX.
4395	* This function is only called once per-VM during initialization.
4396	*
4397	* @returns VBox status code.
4398	* @param pVM The cross context VM structure.
4399	*/
4400	VMMR0DECL(int) VMXR0SetupVM(PVMCC pVM)
4401	{
4402	AssertPtr(pVM);
4403	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4404
4405	LogFlowFunc(("pVM=%p\n", pVM));
4406
4407	/*
4408	* At least verify if VMX is enabled, since we can't check if we're in VMX root mode or not
4409	* without causing a #GP.
4410	*/
4411	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4412	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4413	{ /* likely */ }
4414	else
4415	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4416
4417	/*
4418	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4419	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4420	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4421	*/
4422	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4423	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4424	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4425	{
4426	LogRelFunc(("Invalid real-on-v86 state.\n"));
4427	return VERR_INTERNAL_ERROR;
4428	}
4429
4430	/* Initialize these always, see hmR3InitFinalizeR0().*/
4431	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4432	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4433
4434	/* Setup the tagged-TLB flush handlers. */
4435	int rc = hmR0VmxSetupTaggedTlb(pVM);
4436	if (RT_FAILURE(rc))
4437	{
4438	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4439	return rc;
4440	}
4441
4442	/* Determine LBR capabilities. */
4443	if (pVM->hm.s.vmx.fLbr)
4444	{
4445	rc = hmR0VmxSetupLbrMsrRange(pVM);
4446	if (RT_FAILURE(rc))
4447	{
4448	LogRelFunc(("Failed to setup LBR MSR range. rc=%Rrc\n", rc));
4449	return rc;
4450	}
4451	}
4452
4453	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4454	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4455	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4456	{
4457	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4458	if (RT_SUCCESS(rc))
4459	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4460	else
4461	{
4462	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4463	return rc;
4464	}
4465	}
4466	#endif
4467
4468	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4469	{
4470	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
4471	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4472
4473	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4474	if (RT_SUCCESS(rc))
4475	{
4476	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4477	if (pVM->cpum.ro.GuestFeatures.fVmx)
4478	{
4479	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4480	if (RT_SUCCESS(rc))
4481	{ /* likely */ }
4482	else
4483	{
4484	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4485	return rc;
4486	}
4487	}
4488	#endif
4489	}
4490	else
4491	{
4492	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4493	return rc;
4494	}
4495	}
4496
4497	return VINF_SUCCESS;
4498	}
4499
4500
4501	/**
4502	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4503	* the VMCS.
4504	*/
4505	static void hmR0VmxExportHostControlRegs(void)
4506	{
4507	int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, ASMGetCR0()); AssertRC(rc);
4508	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, ASMGetCR3()); AssertRC(rc);
4509	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, ASMGetCR4()); AssertRC(rc);
4510	}
4511
4512
4513	/**
4514	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4515	* the host-state area in the VMCS.
4516	*
4517	* @returns VBox status code.
4518	* @param pVCpu The cross context virtual CPU structure.
4519	*/
4520	static int hmR0VmxExportHostSegmentRegs(PVMCPUCC pVCpu)
4521	{
4522	/**
4523	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4524	* requirements. See hmR0VmxExportHostSegmentRegs().
4525	*/
4526	#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_selValue) \
4527	if ((a_selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4528	{ \
4529	bool fValidSelector = true; \
4530	if ((a_selValue) & X86_SEL_LDT) \
4531	{ \
4532	uint32_t const uAttr = ASMGetSegAttr(a_selValue); \
4533	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4534	} \
4535	if (fValidSelector) \
4536	{ \
4537	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##a_Seg; \
4538	pVCpu->hm.s.vmx.RestoreHost.uHostSel##a_Seg = (a_selValue); \
4539	} \
4540	(a_selValue) = 0; \
4541	}
4542
4543	/*
4544	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4545	* will be messed up. We should -not- save the messed up state without restoring
4546	* the original host-state, see @bugref{7240}.
4547	*
4548	* This apparently can happen (most likely the FPU changes), deal with it rather than
4549	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4550	*/
4551	if (pVCpu->hm.s.vmx.fRestoreHostFlags > VMX_RESTORE_HOST_REQUIRED)
4552	{
4553	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4554	pVCpu->idCpu));
4555	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4556	}
4557	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4558
4559	/*
4560	* Host segment registers.
4561	*/
4562	RTSEL uSelES = ASMGetES();
4563	RTSEL uSelCS = ASMGetCS();
4564	RTSEL uSelSS = ASMGetSS();
4565	RTSEL uSelDS = ASMGetDS();
4566	RTSEL uSelFS = ASMGetFS();
4567	RTSEL uSelGS = ASMGetGS();
4568	RTSEL uSelTR = ASMGetTR();
4569
4570	/*
4571	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4572	* gain VM-entry and restore them before we get preempted.
4573	*
4574	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4575	*/
4576	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4577	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4578	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4579	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4580
4581	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4582	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4583	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4584	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4585	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4586	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4587	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4588	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4589	Assert(uSelCS);
4590	Assert(uSelTR);
4591
4592	/* Write these host selector fields into the host-state area in the VMCS. */
4593	int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS); AssertRC(rc);
4594	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS); AssertRC(rc);
4595	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS); AssertRC(rc);
4596	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES); AssertRC(rc);
4597	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS); AssertRC(rc);
4598	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS); AssertRC(rc);
4599	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR); AssertRC(rc);
4600
4601	/*
4602	* Host GDTR and IDTR.
4603	*/
4604	RTGDTR Gdtr;
4605	RTIDTR Idtr;
4606	RT_ZERO(Gdtr);
4607	RT_ZERO(Idtr);
4608	ASMGetGDTR(&Gdtr);
4609	ASMGetIDTR(&Idtr);
4610	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRC(rc);
4611	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRC(rc);
4612
4613	/*
4614	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4615	* them to the maximum limit (0xffff) on every VM-exit.
4616	*/
4617	if (Gdtr.cbGdt != 0xffff)
4618	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4619
4620	/*
4621	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4622	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4623	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4624	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4625	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4626	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4627	* at 0xffff on hosts where we are sure it won't cause trouble.
4628	*/
4629	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4630	if (Idtr.cbIdt < 0x0fff)
4631	#else
4632	if (Idtr.cbIdt != 0xffff)
4633	#endif
4634	{
4635	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4636	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4637	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4638	}
4639
4640	/*
4641	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4642	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4643	* RPL should be too in most cases.
4644	*/
4645	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4646	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4647
4648	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4649	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4650
4651	/*
4652	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4653	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4654	* restoration if the host has something else. Task switching is not supported in 64-bit
4655	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4656	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4657	*
4658	* [1] See Intel spec. 3.5 "System Descriptor Types".
4659	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4660	*/
4661	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4662	Assert(pDesc->System.u4Type == 11);
4663	if ( pDesc->System.u16LimitLow != 0x67
4664	\|\| pDesc->System.u4LimitHigh)
4665	{
4666	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4667	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4668	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4669	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4670	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4671	}
4672
4673	/*
4674	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4675	*/
4676	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4677	{
4678	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4679	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4680	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4681	{
4682	/* The GDT is read-only but the writable GDT is available. */
4683	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4684	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4685	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4686	AssertRCReturn(rc, rc);
4687	}
4688	}
4689
4690	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase);
4691	AssertRC(rc);
4692
4693	/*
4694	* Host FS base and GS base.
4695	*/
4696	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4697	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4698	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRC(rc);
4699	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRC(rc);
4700
4701	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4702	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4703	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4704	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4705	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4706
4707	return VINF_SUCCESS;
4708	#undef VMXLOCAL_ADJUST_HOST_SEG
4709	}
4710
4711
4712	/**
4713	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4714	* host-state area of the VMCS.
4715	*
4716	* These MSRs will be automatically restored on the host after every successful
4717	* VM-exit.
4718	*
4719	* @param pVCpu The cross context virtual CPU structure.
4720	*
4721	* @remarks No-long-jump zone!!!
4722	*/
4723	static void hmR0VmxExportHostMsrs(PVMCPUCC pVCpu)
4724	{
4725	AssertPtr(pVCpu);
4726
4727	/*
4728	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4729	* rather than swapping them on every VM-entry.
4730	*/
4731	hmR0VmxLazySaveHostMsrs(pVCpu);
4732
4733	/*
4734	* Host Sysenter MSRs.
4735	*/
4736	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)); AssertRC(rc);
4737	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP)); AssertRC(rc);
4738	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP)); AssertRC(rc);
4739
4740	/*
4741	* Host EFER MSR.
4742	*
4743	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4744	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4745	*/
4746	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4747	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4748	{
4749	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4750	AssertRC(rc);
4751	}
4752
4753	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4754	* hmR0VmxExportGuestEntryExitCtls(). */
4755	}
4756
4757
4758	/**
4759	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4760	*
4761	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4762	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4763	*
4764	* @returns true if we need to load guest EFER, false otherwise.
4765	* @param pVCpu The cross context virtual CPU structure.
4766	* @param pVmxTransient The VMX-transient structure.
4767	*
4768	* @remarks Requires EFER, CR4.
4769	* @remarks No-long-jump zone!!!
4770	*/
4771	static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4772	{
4773	#ifdef HMVMX_ALWAYS_SWAP_EFER
4774	RT_NOREF2(pVCpu, pVmxTransient);
4775	return true;
4776	#else
4777	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4778	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4779	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4780	uint64_t const u64GuestEfer = pCtx->msrEFER;
4781
4782	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4783	/*
4784	* For nested-guests, we shall honor swapping the EFER MSR when requested by
4785	* the nested-guest.
4786	*/
4787	if ( pVmxTransient->fIsNestedGuest
4788	&& ( CPUMIsGuestVmxEntryCtlsSet(pCtx, VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4789	\|\| CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_SAVE_EFER_MSR)
4790	\|\| CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_LOAD_EFER_MSR)))
4791	return true;
4792	# else
4793	RT_NOREF(pVmxTransient);
4794	#endif
4795
4796	/*
4797	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4798	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4799	*/
4800	if ( CPUMIsGuestInLongModeEx(pCtx)
4801	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4802	return true;
4803
4804	/*
4805	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4806	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4807	*
4808	* See Intel spec. 4.5 "IA-32e Paging".
4809	* See Intel spec. 4.1.1 "Three Paging Modes".
4810	*
4811	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4812	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4813	*/
4814	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4815	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4816	if ( (pCtx->cr4 & X86_CR4_PAE)
4817	&& (pCtx->cr0 & X86_CR0_PG))
4818	{
4819	/*
4820	* If nested paging is not used, verify that the guest paging mode matches the
4821	* shadow paging mode which is/will be placed in the VMCS (which is what will
4822	* actually be used while executing the guest and not the CR4 shadow value).
4823	*/
4824	AssertMsg(pVM->hm.s.fNestedPaging \|\| ( pVCpu->hm.s.enmShadowMode == PGMMODE_PAE
4825	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_PAE_NX
4826	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64
4827	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64_NX),
4828	("enmShadowMode=%u\n", pVCpu->hm.s.enmShadowMode));
4829	if ((u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4830	{
4831	/* Verify that the host is NX capable. */
4832	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4833	return true;
4834	}
4835	}
4836
4837	return false;
4838	#endif
4839	}
4840
4841
4842	/**
4843	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4844	* VMCS.
4845	*
4846	* This is typically required when the guest changes paging mode.
4847	*
4848	* @returns VBox status code.
4849	* @param pVCpu The cross context virtual CPU structure.
4850	* @param pVmxTransient The VMX-transient structure.
4851	*
4852	* @remarks Requires EFER.
4853	* @remarks No-long-jump zone!!!
4854	*/
4855	static int hmR0VmxExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4856	{
4857	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4858	{
4859	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4860	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4861
4862	/*
4863	* VM-entry controls.
4864	*/
4865	{
4866	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4867	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4868
4869	/*
4870	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4871	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4872	*
4873	* For nested-guests, this is a mandatory VM-entry control. It's also
4874	* required because we do not want to leak host bits to the nested-guest.
4875	*/
4876	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4877
4878	/*
4879	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4880	*
4881	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4882	* required to get the nested-guest working with hardware-assisted VMX execution.
4883	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
4884	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4885	* here rather than while merging the guest VMCS controls.
4886	*/
4887	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4888	{
4889	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
4890	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4891	}
4892	else
4893	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4894
4895	/*
4896	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4897	*
4898	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4899	* regardless of whether the nested-guest VMCS specifies it because we are free to
4900	* load whatever MSRs we require and we do not need to modify the guest visible copy
4901	* of the VM-entry MSR load area.
4902	*/
4903	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4904	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4905	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4906	else
4907	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4908
4909	/*
4910	* The following should -not- be set (since we're not in SMM mode):
4911	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4912	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4913	*/
4914
4915	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4916	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4917
4918	if ((fVal & fZap) == fVal)
4919	{ /* likely */ }
4920	else
4921	{
4922	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4923	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4924	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4925	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4926	}
4927
4928	/* Commit it to the VMCS. */
4929	if (pVmcsInfo->u32EntryCtls != fVal)
4930	{
4931	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4932	AssertRC(rc);
4933	pVmcsInfo->u32EntryCtls = fVal;
4934	}
4935	}
4936
4937	/*
4938	* VM-exit controls.
4939	*/
4940	{
4941	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4942	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4943
4944	/*
4945	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4946	* supported the 1-setting of this bit.
4947	*
4948	* For nested-guests, we set the "save debug controls" as the converse
4949	* "load debug controls" is mandatory for nested-guests anyway.
4950	*/
4951	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4952
4953	/*
4954	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4955	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4956	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4957	* hmR0VmxExportHostMsrs().
4958	*
4959	* For nested-guests, we always set this bit as we do not support 32-bit
4960	* hosts.
4961	*/
4962	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4963
4964	/*
4965	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4966	*
4967	* For nested-guests, we should use the "save IA32_EFER" control if we also
4968	* used the "load IA32_EFER" control while exporting VM-entry controls.
4969	*/
4970	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4971	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4972	{
4973	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4974	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4975	}
4976
4977	/*
4978	* Enable saving of the VMX-preemption timer value on VM-exit.
4979	* For nested-guests, currently not exposed/used.
4980	*/
4981	if ( pVM->hm.s.vmx.fUsePreemptTimer
4982	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4983	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4984
4985	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4986	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4987
4988	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4989	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4990	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4991
4992	if ((fVal & fZap) == fVal)
4993	{ /* likely */ }
4994	else
4995	{
4996	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4997	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4998	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4999	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5000	}
5001
5002	/* Commit it to the VMCS. */
5003	if (pVmcsInfo->u32ExitCtls != fVal)
5004	{
5005	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5006	AssertRC(rc);
5007	pVmcsInfo->u32ExitCtls = fVal;
5008	}
5009	}
5010
5011	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5012	}
5013	return VINF_SUCCESS;
5014	}
5015
5016
5017	/**
5018	* Sets the TPR threshold in the VMCS.
5019	*
5020	* @param pVmcsInfo The VMCS info. object.
5021	* @param u32TprThreshold The TPR threshold (task-priority class only).
5022	*/
5023	DECLINLINE(void) hmR0VmxApicSetTprThreshold(PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5024	{
5025	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5026	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5027	RT_NOREF(pVmcsInfo);
5028	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5029	AssertRC(rc);
5030	}
5031
5032
5033	/**
5034	* Exports the guest APIC TPR state into the VMCS.
5035	*
5036	* @param pVCpu The cross context virtual CPU structure.
5037	* @param pVmxTransient The VMX-transient structure.
5038	*
5039	* @remarks No-long-jump zone!!!
5040	*/
5041	static void hmR0VmxExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5042	{
5043	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5044	{
5045	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5046
5047	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5048	if (!pVmxTransient->fIsNestedGuest)
5049	{
5050	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5051	&& APICIsEnabled(pVCpu))
5052	{
5053	/*
5054	* Setup TPR shadowing.
5055	*/
5056	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5057	{
5058	bool fPendingIntr = false;
5059	uint8_t u8Tpr = 0;
5060	uint8_t u8PendingIntr = 0;
5061	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5062	AssertRC(rc);
5063
5064	/*
5065	* If there are interrupts pending but masked by the TPR, instruct VT-x to
5066	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5067	* priority of the pending interrupt so we can deliver the interrupt. If there
5068	* are no interrupts pending, set threshold to 0 to not cause any
5069	* TPR-below-threshold VM-exits.
5070	*/
5071	uint32_t u32TprThreshold = 0;
5072	if (fPendingIntr)
5073	{
5074	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5075	(which is the Task-Priority Class). */
5076	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5077	const uint8_t u8TprPriority = u8Tpr >> 4;
5078	if (u8PendingPriority <= u8TprPriority)
5079	u32TprThreshold = u8PendingPriority;
5080	}
5081
5082	hmR0VmxApicSetTprThreshold(pVmcsInfo, u32TprThreshold);
5083	}
5084	}
5085	}
5086	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5087	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5088	}
5089	}
5090
5091
5092	/**
5093	* Gets the guest interruptibility-state and updates related force-flags.
5094	*
5095	* @returns Guest's interruptibility-state.
5096	* @param pVCpu The cross context virtual CPU structure.
5097	*
5098	* @remarks No-long-jump zone!!!
5099	*/
5100	static uint32_t hmR0VmxGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu)
5101	{
5102	/*
5103	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5104	*/
5105	uint32_t fIntrState = 0;
5106	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5107	{
5108	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
5109	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
5110
5111	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5112	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5113	{
5114	if (pCtx->eflags.Bits.u1IF)
5115	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5116	else
5117	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5118	}
5119	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5120	{
5121	/*
5122	* We can clear the inhibit force flag as even if we go back to the recompiler
5123	* without executing guest code in VT-x, the flag's condition to be cleared is
5124	* met and thus the cleared state is correct.
5125	*/
5126	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5127	}
5128	}
5129
5130	/*
5131	* Check if we should inhibit NMI delivery.
5132	*/
5133	if (CPUMIsGuestNmiBlocking(pVCpu))
5134	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5135
5136	/*
5137	* Validate.
5138	*/
5139	#ifdef VBOX_STRICT
5140	/* We don't support block-by-SMI yet.*/
5141	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
5142
5143	/* Block-by-STI must not be set when interrupts are disabled. */
5144	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5145	{
5146	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5147	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
5148	}
5149	#endif
5150
5151	return fIntrState;
5152	}
5153
5154
5155	/**
5156	* Exports the exception intercepts required for guest execution in the VMCS.
5157	*
5158	* @param pVCpu The cross context virtual CPU structure.
5159	* @param pVmxTransient The VMX-transient structure.
5160	*
5161	* @remarks No-long-jump zone!!!
5162	*/
5163	static void hmR0VmxExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5164	{
5165	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5166	{
5167	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5168	if ( !pVmxTransient->fIsNestedGuest
5169	&& pVCpu->hm.s.fGIMTrapXcptUD)
5170	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5171	else
5172	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5173
5174	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5175	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5176	}
5177	}
5178
5179
5180	/**
5181	* Exports the guest's RIP into the guest-state area in the VMCS.
5182	*
5183	* @param pVCpu The cross context virtual CPU structure.
5184	*
5185	* @remarks No-long-jump zone!!!
5186	*/
5187	static void hmR0VmxExportGuestRip(PVMCPUCC pVCpu)
5188	{
5189	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5190	{
5191	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5192
5193	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5194	AssertRC(rc);
5195
5196	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5197	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5198	}
5199	}
5200
5201
5202	/**
5203	* Exports the guest's RSP into the guest-state area in the VMCS.
5204	*
5205	* @param pVCpu The cross context virtual CPU structure.
5206	*
5207	* @remarks No-long-jump zone!!!
5208	*/
5209	static void hmR0VmxExportGuestRsp(PVMCPUCC pVCpu)
5210	{
5211	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5212	{
5213	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5214
5215	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5216	AssertRC(rc);
5217
5218	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5219	Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
5220	}
5221	}
5222
5223
5224	/**
5225	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5226	*
5227	* @param pVCpu The cross context virtual CPU structure.
5228	* @param pVmxTransient The VMX-transient structure.
5229	*
5230	* @remarks No-long-jump zone!!!
5231	*/
5232	static void hmR0VmxExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5233	{
5234	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5235	{
5236	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5237
5238	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5239	Let us assert it as such and use 32-bit VMWRITE. */
5240	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5241	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5242	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5243	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5244
5245	/*
5246	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5247	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5248	* can run the real-mode guest code under Virtual 8086 mode.
5249	*/
5250	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5251	if (pVmcsInfo->RealMode.fRealOnV86Active)
5252	{
5253	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5254	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5255	Assert(!pVmxTransient->fIsNestedGuest);
5256	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5257	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5258	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5259	}
5260
5261	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5262	AssertRC(rc);
5263
5264	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5265	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5266	}
5267	}
5268
5269
5270	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5271	/**
5272	* Copies the nested-guest VMCS to the shadow VMCS.
5273	*
5274	* @returns VBox status code.
5275	* @param pVCpu The cross context virtual CPU structure.
5276	* @param pVmcsInfo The VMCS info. object.
5277	*
5278	* @remarks No-long-jump zone!!!
5279	*/
5280	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5281	{
5282	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5283	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5284
5285	/*
5286	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5287	* current VMCS, as we may try saving guest lazy MSRs.
5288	*
5289	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5290	* calling the import VMCS code which is currently performing the guest MSR reads
5291	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5292	* and the rest of the VMX leave session machinery.
5293	*/
5294	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5295
5296	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5297	if (RT_SUCCESS(rc))
5298	{
5299	/*
5300	* Copy all guest read/write VMCS fields.
5301	*
5302	* We don't check for VMWRITE failures here for performance reasons and
5303	* because they are not expected to fail, barring irrecoverable conditions
5304	* like hardware errors.
5305	*/
5306	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5307	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5308	{
5309	uint64_t u64Val;
5310	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5311	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5312	VMXWriteVmcs64(uVmcsField, u64Val);
5313	}
5314
5315	/*
5316	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5317	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5318	*/
5319	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5320	{
5321	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5322	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5323	{
5324	uint64_t u64Val;
5325	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5326	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5327	VMXWriteVmcs64(uVmcsField, u64Val);
5328	}
5329	}
5330
5331	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5332	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5333	}
5334
5335	ASMSetFlags(fEFlags);
5336	return rc;
5337	}
5338
5339
5340	/**
5341	* Copies the shadow VMCS to the nested-guest VMCS.
5342	*
5343	* @returns VBox status code.
5344	* @param pVCpu The cross context virtual CPU structure.
5345	* @param pVmcsInfo The VMCS info. object.
5346	*
5347	* @remarks Called with interrupts disabled.
5348	*/
5349	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5350	{
5351	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5352	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5353	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5354
5355	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5356	if (RT_SUCCESS(rc))
5357	{
5358	/*
5359	* Copy guest read/write fields from the shadow VMCS.
5360	* Guest read-only fields cannot be modified, so no need to copy them.
5361	*
5362	* We don't check for VMREAD failures here for performance reasons and
5363	* because they are not expected to fail, barring irrecoverable conditions
5364	* like hardware errors.
5365	*/
5366	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5367	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5368	{
5369	uint64_t u64Val;
5370	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5371	VMXReadVmcs64(uVmcsField, &u64Val);
5372	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5373	}
5374
5375	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5376	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5377	}
5378	return rc;
5379	}
5380
5381
5382	/**
5383	* Enables VMCS shadowing for the given VMCS info. object.
5384	*
5385	* @param pVmcsInfo The VMCS info. object.
5386	*
5387	* @remarks No-long-jump zone!!!
5388	*/
5389	static void hmR0VmxEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5390	{
5391	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5392	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5393	{
5394	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5395	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5396	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5397	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
5398	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5399	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5400	Log4Func(("Enabled\n"));
5401	}
5402	}
5403
5404
5405	/**
5406	* Disables VMCS shadowing for the given VMCS info. object.
5407	*
5408	* @param pVmcsInfo The VMCS info. object.
5409	*
5410	* @remarks No-long-jump zone!!!
5411	*/
5412	static void hmR0VmxDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5413	{
5414	/*
5415	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5416	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5417	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5418	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5419	*
5420	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5421	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5422	*/
5423	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5424	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5425	{
5426	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5427	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5428	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
5429	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5430	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5431	Log4Func(("Disabled\n"));
5432	}
5433	}
5434	#endif
5435
5436
5437	/**
5438	* Exports the guest hardware-virtualization state.
5439	*
5440	* @returns VBox status code.
5441	* @param pVCpu The cross context virtual CPU structure.
5442	* @param pVmxTransient The VMX-transient structure.
5443	*
5444	* @remarks No-long-jump zone!!!
5445	*/
5446	static int hmR0VmxExportGuestHwvirtState(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5447	{
5448	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5449	{
5450	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5451	/*
5452	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5453	* VMCS shadowing.
5454	*/
5455	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5456	{
5457	/*
5458	* If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
5459	* copy the nested hypervisor's current VMCS into the shadow VMCS and enable
5460	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5461	*
5462	* We check for VMX root mode here in case the guest executes VMXOFF without
5463	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5464	* not clear the current VMCS pointer.
5465	*/
5466	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5467	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5468	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5469	&& CPUMIsGuestVmxCurrentVmcsValid(&pVCpu->cpum.GstCtx))
5470	{
5471	/* Paranoia. */
5472	Assert(!pVmxTransient->fIsNestedGuest);
5473
5474	/*
5475	* For performance reasons, also check if the nested hypervisor's current VMCS
5476	* was newly loaded or modified before copying it to the shadow VMCS.
5477	*/
5478	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5479	{
5480	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5481	AssertRCReturn(rc, rc);
5482	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5483	}
5484	hmR0VmxEnableVmcsShadowing(pVmcsInfo);
5485	}
5486	else
5487	hmR0VmxDisableVmcsShadowing(pVmcsInfo);
5488	}
5489	#else
5490	NOREF(pVmxTransient);
5491	#endif
5492	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5493	}
5494	return VINF_SUCCESS;
5495	}
5496
5497
5498	/**
5499	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5500	*
5501	* The guest FPU state is always pre-loaded hence we don't need to bother about
5502	* sharing FPU related CR0 bits between the guest and host.
5503	*
5504	* @returns VBox status code.
5505	* @param pVCpu The cross context virtual CPU structure.
5506	* @param pVmxTransient The VMX-transient structure.
5507	*
5508	* @remarks No-long-jump zone!!!
5509	*/
5510	static int hmR0VmxExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5511	{
5512	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5513	{
5514	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5515	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5516
5517	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0;
5518	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5519	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5520	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5521	else
5522	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5523
5524	if (!pVmxTransient->fIsNestedGuest)
5525	{
5526	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5527	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5528	uint64_t const u64ShadowCr0 = u64GuestCr0;
5529	Assert(!RT_HI_U32(u64GuestCr0));
5530
5531	/*
5532	* Setup VT-x's view of the guest CR0.
5533	*/
5534	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5535	if (pVM->hm.s.fNestedPaging)
5536	{
5537	if (CPUMIsGuestPagingEnabled(pVCpu))
5538	{
5539	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5540	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5541	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5542	}
5543	else
5544	{
5545	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5546	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5547	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5548	}
5549
5550	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5551	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5552	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5553	}
5554	else
5555	{
5556	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5557	u64GuestCr0 \|= X86_CR0_WP;
5558	}
5559
5560	/*
5561	* Guest FPU bits.
5562	*
5563	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5564	* using CR0.TS.
5565	*
5566	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5567	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5568	*/
5569	u64GuestCr0 \|= X86_CR0_NE;
5570
5571	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5572	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5573
5574	/*
5575	* Update exception intercepts.
5576	*/
5577	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5578	if (pVmcsInfo->RealMode.fRealOnV86Active)
5579	{
5580	Assert(PDMVmmDevHeapIsEnabled(pVM));
5581	Assert(pVM->hm.s.vmx.pRealModeTSS);
5582	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5583	}
5584	else
5585	{
5586	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5587	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5588	if (fInterceptMF)
5589	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5590	}
5591
5592	/* Additional intercepts for debugging, define these yourself explicitly. */
5593	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5594	uXcptBitmap \|= 0
5595	\| RT_BIT(X86_XCPT_BP)
5596	\| RT_BIT(X86_XCPT_DE)
5597	\| RT_BIT(X86_XCPT_NM)
5598	\| RT_BIT(X86_XCPT_TS)
5599	\| RT_BIT(X86_XCPT_UD)
5600	\| RT_BIT(X86_XCPT_NP)
5601	\| RT_BIT(X86_XCPT_SS)
5602	\| RT_BIT(X86_XCPT_GP)
5603	\| RT_BIT(X86_XCPT_PF)
5604	\| RT_BIT(X86_XCPT_MF)
5605	;
5606	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5607	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5608	#endif
5609	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5610	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5611	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5612
5613	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5614	u64GuestCr0 \|= fSetCr0;
5615	u64GuestCr0 &= fZapCr0;
5616	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5617
5618	/* Commit the CR0 and related fields to the guest VMCS. */
5619	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5620	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5621	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5622	{
5623	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5624	AssertRC(rc);
5625	}
5626	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5627	{
5628	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5629	AssertRC(rc);
5630	}
5631
5632	/* Update our caches. */
5633	pVmcsInfo->u32ProcCtls = uProcCtls;
5634	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5635
5636	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5637	}
5638	else
5639	{
5640	/*
5641	* With nested-guests, we may have extended the guest/host mask here since we
5642	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5643	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
5644	* originally supplied. We must copy those bits from the nested-guest CR0 into
5645	* the nested-guest CR0 read-shadow.
5646	*/
5647	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5648	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5649	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5650	Assert(!RT_HI_U32(u64GuestCr0));
5651	Assert(u64GuestCr0 & X86_CR0_NE);
5652
5653	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5654	u64GuestCr0 \|= fSetCr0;
5655	u64GuestCr0 &= fZapCr0;
5656	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5657
5658	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5659	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5660	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5661
5662	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5663	}
5664
5665	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5666	}
5667
5668	return VINF_SUCCESS;
5669	}
5670
5671
5672	/**
5673	* Exports the guest control registers (CR3, CR4) into the guest-state area
5674	* in the VMCS.
5675	*
5676	* @returns VBox strict status code.
5677	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5678	* without unrestricted guest access and the VMMDev is not presently
5679	* mapped (e.g. EFI32).
5680	*
5681	* @param pVCpu The cross context virtual CPU structure.
5682	* @param pVmxTransient The VMX-transient structure.
5683	*
5684	* @remarks No-long-jump zone!!!
5685	*/
5686	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
5687	{
5688	int rc = VINF_SUCCESS;
5689	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5690
5691	/*
5692	* Guest CR2.
5693	* It's always loaded in the assembler code. Nothing to do here.
5694	*/
5695
5696	/*
5697	* Guest CR3.
5698	*/
5699	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5700	{
5701	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5702
5703	if (pVM->hm.s.fNestedPaging)
5704	{
5705	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5706	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5707
5708	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5709	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5710	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5711	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5712
5713	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5714	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5715	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5716
5717	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5718	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5719	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5720	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5721	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5722	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5723	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5724
5725	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5726	AssertRC(rc);
5727
5728	uint64_t u64GuestCr3;
5729	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5730	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5731	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5732	{
5733	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5734	if (CPUMIsGuestInPAEModeEx(pCtx))
5735	{
5736	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5737	AssertRC(rc);
5738	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
5739	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
5740	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
5741	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
5742	}
5743
5744	/*
5745	* The guest's view of its CR3 is unblemished with nested paging when the
5746	* guest is using paging or we have unrestricted guest execution to handle
5747	* the guest when it's not using paging.
5748	*/
5749	u64GuestCr3 = pCtx->cr3;
5750	}
5751	else
5752	{
5753	/*
5754	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5755	* thinks it accesses physical memory directly, we use our identity-mapped
5756	* page table to map guest-linear to guest-physical addresses. EPT takes care
5757	* of translating it to host-physical addresses.
5758	*/
5759	RTGCPHYS GCPhys;
5760	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5761
5762	/* We obtain it here every time as the guest could have relocated this PCI region. */
5763	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5764	if (RT_SUCCESS(rc))
5765	{ /* likely */ }
5766	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5767	{
5768	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5769	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5770	}
5771	else
5772	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5773
5774	u64GuestCr3 = GCPhys;
5775	}
5776
5777	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
5778	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, u64GuestCr3);
5779	AssertRC(rc);
5780	}
5781	else
5782	{
5783	Assert(!pVmxTransient->fIsNestedGuest);
5784	/* Non-nested paging case, just use the hypervisor's CR3. */
5785	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5786
5787	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
5788	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5789	AssertRC(rc);
5790	}
5791
5792	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5793	}
5794
5795	/*
5796	* Guest CR4.
5797	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5798	*/
5799	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5800	{
5801	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5802	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5803
5804	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0;
5805	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5806
5807	/*
5808	* With nested-guests, we may have extended the guest/host mask here (since we
5809	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5810	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5811	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
5812	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5813	*/
5814	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5815	uint64_t u64GuestCr4 = pCtx->cr4;
5816	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5817	? pCtx->cr4
5818	: CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
5819	Assert(!RT_HI_U32(u64GuestCr4));
5820
5821	/*
5822	* Setup VT-x's view of the guest CR4.
5823	*
5824	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5825	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5826	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5827	*
5828	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5829	*/
5830	if (pVmcsInfo->RealMode.fRealOnV86Active)
5831	{
5832	Assert(pVM->hm.s.vmx.pRealModeTSS);
5833	Assert(PDMVmmDevHeapIsEnabled(pVM));
5834	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5835	}
5836
5837	if (pVM->hm.s.fNestedPaging)
5838	{
5839	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5840	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5841	{
5842	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5843	u64GuestCr4 \|= X86_CR4_PSE;
5844	/* Our identity mapping is a 32-bit page directory. */
5845	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5846	}
5847	/* else use guest CR4.*/
5848	}
5849	else
5850	{
5851	Assert(!pVmxTransient->fIsNestedGuest);
5852
5853	/*
5854	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5855	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5856	*/
5857	switch (pVCpu->hm.s.enmShadowMode)
5858	{
5859	case PGMMODE_REAL: /* Real-mode. */
5860	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5861	case PGMMODE_32_BIT: /* 32-bit paging. */
5862	{
5863	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5864	break;
5865	}
5866
5867	case PGMMODE_PAE: /* PAE paging. */
5868	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5869	{
5870	u64GuestCr4 \|= X86_CR4_PAE;
5871	break;
5872	}
5873
5874	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5875	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5876	{
5877	#ifdef VBOX_WITH_64_BITS_GUESTS
5878	/* For our assumption in hmR0VmxShouldSwapEferMsr. */
5879	Assert(u64GuestCr4 & X86_CR4_PAE);
5880	break;
5881	#endif
5882	}
5883	default:
5884	AssertFailed();
5885	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5886	}
5887	}
5888
5889	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
5890	u64GuestCr4 \|= fSetCr4;
5891	u64GuestCr4 &= fZapCr4;
5892
5893	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5894	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
5895	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
5896
5897	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5898	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5899
5900	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5901
5902	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5903	}
5904	return rc;
5905	}
5906
5907
5908	/**
5909	* Exports the guest debug registers into the guest-state area in the VMCS.
5910	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5911	*
5912	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5913	*
5914	* @returns VBox status code.
5915	* @param pVCpu The cross context virtual CPU structure.
5916	* @param pVmxTransient The VMX-transient structure.
5917	*
5918	* @remarks No-long-jump zone!!!
5919	*/
5920	static int hmR0VmxExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5921	{
5922	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5923
5924	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5925	* stepping. */
5926	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5927	if (pVmxTransient->fIsNestedGuest)
5928	{
5929	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5930	AssertRC(rc);
5931
5932	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5933	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5934	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5935	AssertRC(rc);
5936	return VINF_SUCCESS;
5937	}
5938
5939	#ifdef VBOX_STRICT
5940	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5941	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5942	{
5943	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5944	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5945	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5946	}
5947	#endif
5948
5949	bool fSteppingDB = false;
5950	bool fInterceptMovDRx = false;
5951	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5952	if (pVCpu->hm.s.fSingleInstruction)
5953	{
5954	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5955	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5956	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5957	{
5958	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5959	Assert(fSteppingDB == false);
5960	}
5961	else
5962	{
5963	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5964	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5965	pVCpu->hm.s.fClearTrapFlag = true;
5966	fSteppingDB = true;
5967	}
5968	}
5969
5970	uint64_t u64GuestDr7;
5971	if ( fSteppingDB
5972	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5973	{
5974	/*
5975	* Use the combined guest and host DRx values found in the hypervisor register set
5976	* because the hypervisor debugger has breakpoints active or someone is single stepping
5977	* on the host side without a monitor trap flag.
5978	*
5979	* Note! DBGF expects a clean DR6 state before executing guest code.
5980	*/
5981	if (!CPUMIsHyperDebugStateActive(pVCpu))
5982	{
5983	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5984	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5985	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5986	}
5987
5988	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5989	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
5990	pVCpu->hm.s.fUsingHyperDR7 = true;
5991	fInterceptMovDRx = true;
5992	}
5993	else
5994	{
5995	/*
5996	* If the guest has enabled debug registers, we need to load them prior to
5997	* executing guest code so they'll trigger at the right time.
5998	*/
5999	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
6000	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
6001	{
6002	if (!CPUMIsGuestDebugStateActive(pVCpu))
6003	{
6004	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6005	Assert(CPUMIsGuestDebugStateActive(pVCpu));
6006	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6007	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6008	}
6009	Assert(!fInterceptMovDRx);
6010	}
6011	else if (!CPUMIsGuestDebugStateActive(pVCpu))
6012	{
6013	/*
6014	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
6015	* must intercept #DB in order to maintain a correct DR6 guest value, and
6016	* because we need to intercept it to prevent nested #DBs from hanging the
6017	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
6018	*/
6019	fInterceptMovDRx = true;
6020	}
6021
6022	/* Update DR7 with the actual guest value. */
6023	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
6024	pVCpu->hm.s.fUsingHyperDR7 = false;
6025	}
6026
6027	if (fInterceptMovDRx)
6028	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
6029	else
6030	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
6031
6032	/*
6033	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
6034	* monitor-trap flag and update our cache.
6035	*/
6036	if (uProcCtls != pVmcsInfo->u32ProcCtls)
6037	{
6038	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6039	AssertRC(rc);
6040	pVmcsInfo->u32ProcCtls = uProcCtls;
6041	}
6042
6043	/*
6044	* Update guest DR7.
6045	*/
6046	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
6047	AssertRC(rc);
6048
6049	/*
6050	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6051	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6052	*
6053	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6054	*/
6055	if (fSteppingDB)
6056	{
6057	Assert(pVCpu->hm.s.fSingleInstruction);
6058	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6059
6060	uint32_t fIntrState = 0;
6061	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6062	AssertRC(rc);
6063
6064	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6065	{
6066	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6067	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6068	AssertRC(rc);
6069	}
6070	}
6071
6072	return VINF_SUCCESS;
6073	}
6074
6075
6076	#ifdef VBOX_STRICT
6077	/**
6078	* Strict function to validate segment registers.
6079	*
6080	* @param pVCpu The cross context virtual CPU structure.
6081	* @param pVmcsInfo The VMCS info. object.
6082	*
6083	* @remarks Will import guest CR0 on strict builds during validation of
6084	* segments.
6085	*/
6086	static void hmR0VmxValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
6087	{
6088	/*
6089	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6090	*
6091	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6092	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6093	* unusable bit and doesn't change the guest-context value.
6094	*/
6095	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6096	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6097	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6098	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6099	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6100	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6101	{
6102	/* Protected mode checks */
6103	/* CS */
6104	Assert(pCtx->cs.Attr.n.u1Present);
6105	Assert(!(pCtx->cs.Attr.u & 0xf00));
6106	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6107	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6108	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6109	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6110	\|\| (pCtx->cs.Attr.n.u1Granularity));
6111	/* CS cannot be loaded with NULL in protected mode. */
6112	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6113	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6114	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6115	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6116	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6117	else
6118	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6119	/* SS */
6120	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6121	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6122	if ( !(pCtx->cr0 & X86_CR0_PE)
6123	\|\| pCtx->cs.Attr.n.u4Type == 3)
6124	{
6125	Assert(!pCtx->ss.Attr.n.u2Dpl);
6126	}
6127	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6128	{
6129	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6130	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6131	Assert(pCtx->ss.Attr.n.u1Present);
6132	Assert(!(pCtx->ss.Attr.u & 0xf00));
6133	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6134	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6135	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6136	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6137	\|\| (pCtx->ss.Attr.n.u1Granularity));
6138	}
6139	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6140	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6141	{
6142	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6143	Assert(pCtx->ds.Attr.n.u1Present);
6144	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6145	Assert(!(pCtx->ds.Attr.u & 0xf00));
6146	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6147	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6148	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6149	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6150	\|\| (pCtx->ds.Attr.n.u1Granularity));
6151	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6152	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6153	}
6154	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6155	{
6156	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6157	Assert(pCtx->es.Attr.n.u1Present);
6158	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6159	Assert(!(pCtx->es.Attr.u & 0xf00));
6160	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6161	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6162	\|\| !(pCtx->es.Attr.n.u1Granularity));
6163	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6164	\|\| (pCtx->es.Attr.n.u1Granularity));
6165	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6166	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6167	}
6168	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6169	{
6170	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6171	Assert(pCtx->fs.Attr.n.u1Present);
6172	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6173	Assert(!(pCtx->fs.Attr.u & 0xf00));
6174	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6175	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6176	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6177	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6178	\|\| (pCtx->fs.Attr.n.u1Granularity));
6179	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6180	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6181	}
6182	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6183	{
6184	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6185	Assert(pCtx->gs.Attr.n.u1Present);
6186	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6187	Assert(!(pCtx->gs.Attr.u & 0xf00));
6188	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6189	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6190	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6191	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6192	\|\| (pCtx->gs.Attr.n.u1Granularity));
6193	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6194	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6195	}
6196	/* 64-bit capable CPUs. */
6197	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6198	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6199	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6200	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6201	}
6202	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6203	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6204	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6205	{
6206	/* Real and v86 mode checks. */
6207	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6208	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6209	if (pVmcsInfo->RealMode.fRealOnV86Active)
6210	{
6211	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6212	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6213	}
6214	else
6215	{
6216	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6217	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6218	}
6219
6220	/* CS */
6221	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6222	Assert(pCtx->cs.u32Limit == 0xffff);
6223	Assert(u32CSAttr == 0xf3);
6224	/* SS */
6225	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6226	Assert(pCtx->ss.u32Limit == 0xffff);
6227	Assert(u32SSAttr == 0xf3);
6228	/* DS */
6229	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6230	Assert(pCtx->ds.u32Limit == 0xffff);
6231	Assert(u32DSAttr == 0xf3);
6232	/* ES */
6233	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6234	Assert(pCtx->es.u32Limit == 0xffff);
6235	Assert(u32ESAttr == 0xf3);
6236	/* FS */
6237	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6238	Assert(pCtx->fs.u32Limit == 0xffff);
6239	Assert(u32FSAttr == 0xf3);
6240	/* GS */
6241	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6242	Assert(pCtx->gs.u32Limit == 0xffff);
6243	Assert(u32GSAttr == 0xf3);
6244	/* 64-bit capable CPUs. */
6245	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6246	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6247	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6248	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6249	}
6250	}
6251	#endif /* VBOX_STRICT */
6252
6253
6254	/**
6255	* Exports a guest segment register into the guest-state area in the VMCS.
6256	*
6257	* @returns VBox status code.
6258	* @param pVCpu The cross context virtual CPU structure.
6259	* @param pVmcsInfo The VMCS info. object.
6260	* @param iSegReg The segment register number (X86_SREG_XXX).
6261	* @param pSelReg Pointer to the segment selector.
6262	*
6263	* @remarks No-long-jump zone!!!
6264	*/
6265	static int hmR0VmxExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
6266	{
6267	Assert(iSegReg < X86_SREG_COUNT);
6268
6269	uint32_t u32Access = pSelReg->Attr.u;
6270	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6271	{
6272	/*
6273	* The way to differentiate between whether this is really a null selector or was just
6274	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6275	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6276	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6277	* NULL selectors loaded in protected-mode have their attribute as 0.
6278	*/
6279	if (u32Access)
6280	{ }
6281	else
6282	u32Access = X86DESCATTR_UNUSABLE;
6283	}
6284	else
6285	{
6286	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6287	u32Access = 0xf3;
6288	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6289	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6290	RT_NOREF_PV(pVCpu);
6291	}
6292
6293	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6294	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6295	("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));
6296
6297	/*
6298	* Commit it to the VMCS.
6299	*/
6300	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
6301	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
6302	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
6303	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
6304	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_SEG_SEL(iSegReg), pSelReg->Sel); AssertRC(rc);
6305	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), pSelReg->u32Limit); AssertRC(rc);
6306	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SEG_BASE(iSegReg), pSelReg->u64Base); AssertRC(rc);
6307	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access); AssertRC(rc);
6308	return VINF_SUCCESS;
6309	}
6310
6311
6312	/**
6313	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6314	* area in the VMCS.
6315	*
6316	* @returns VBox status code.
6317	* @param pVCpu The cross context virtual CPU structure.
6318	* @param pVmxTransient The VMX-transient structure.
6319	*
6320	* @remarks Will import guest CR0 on strict builds during validation of
6321	* segments.
6322	* @remarks No-long-jump zone!!!
6323	*/
6324	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6325	{
6326	int rc = VERR_INTERNAL_ERROR_5;
6327	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6328	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6329	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6330
6331	/*
6332	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6333	*/
6334	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6335	{
6336	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6337	{
6338	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6339	if (pVmcsInfo->RealMode.fRealOnV86Active)
6340	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6341	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6342	AssertRC(rc);
6343	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6344	}
6345
6346	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6347	{
6348	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6349	if (pVmcsInfo->RealMode.fRealOnV86Active)
6350	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6351	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6352	AssertRC(rc);
6353	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6354	}
6355
6356	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6357	{
6358	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6359	if (pVmcsInfo->RealMode.fRealOnV86Active)
6360	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6361	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6362	AssertRC(rc);
6363	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6364	}
6365
6366	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6367	{
6368	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6369	if (pVmcsInfo->RealMode.fRealOnV86Active)
6370	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6371	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6372	AssertRC(rc);
6373	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6374	}
6375
6376	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6377	{
6378	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6379	if (pVmcsInfo->RealMode.fRealOnV86Active)
6380	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6381	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6382	AssertRC(rc);
6383	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6384	}
6385
6386	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6387	{
6388	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6389	if (pVmcsInfo->RealMode.fRealOnV86Active)
6390	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6391	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6392	AssertRC(rc);
6393	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6394	}
6395
6396	#ifdef VBOX_STRICT
6397	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6398	#endif
6399	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6400	pCtx->cs.Attr.u));
6401	}
6402
6403	/*
6404	* Guest TR.
6405	*/
6406	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6407	{
6408	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6409
6410	/*
6411	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6412	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6413	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6414	*/
6415	uint16_t u16Sel;
6416	uint32_t u32Limit;
6417	uint64_t u64Base;
6418	uint32_t u32AccessRights;
6419	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6420	{
6421	u16Sel = pCtx->tr.Sel;
6422	u32Limit = pCtx->tr.u32Limit;
6423	u64Base = pCtx->tr.u64Base;
6424	u32AccessRights = pCtx->tr.Attr.u;
6425	}
6426	else
6427	{
6428	Assert(!pVmxTransient->fIsNestedGuest);
6429	Assert(pVM->hm.s.vmx.pRealModeTSS);
6430	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6431
6432	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6433	RTGCPHYS GCPhys;
6434	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6435	AssertRCReturn(rc, rc);
6436
6437	X86DESCATTR DescAttr;
6438	DescAttr.u = 0;
6439	DescAttr.n.u1Present = 1;
6440	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6441
6442	u16Sel = 0;
6443	u32Limit = HM_VTX_TSS_SIZE;
6444	u64Base = GCPhys;
6445	u32AccessRights = DescAttr.u;
6446	}
6447
6448	/* Validate. */
6449	Assert(!(u16Sel & RT_BIT(2)));
6450	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6451	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6452	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6453	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6454	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6455	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6456	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6457	Assert( (u32Limit & 0xfff) == 0xfff
6458	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6459	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6460	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6461
6462	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
6463	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
6464	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
6465	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
6466
6467	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6468	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6469	}
6470
6471	/*
6472	* Guest GDTR.
6473	*/
6474	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6475	{
6476	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6477
6478	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
6479	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
6480
6481	/* Validate. */
6482	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6483
6484	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6485	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6486	}
6487
6488	/*
6489	* Guest LDTR.
6490	*/
6491	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6492	{
6493	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6494
6495	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6496	uint32_t u32Access;
6497	if ( !pVmxTransient->fIsNestedGuest
6498	&& !pCtx->ldtr.Attr.u)
6499	u32Access = X86DESCATTR_UNUSABLE;
6500	else
6501	u32Access = pCtx->ldtr.Attr.u;
6502
6503	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
6504	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
6505	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
6506	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
6507
6508	/* Validate. */
6509	if (!(u32Access & X86DESCATTR_UNUSABLE))
6510	{
6511	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6512	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6513	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6514	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6515	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6516	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6517	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6518	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6519	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6520	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6521	}
6522
6523	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6524	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6525	}
6526
6527	/*
6528	* Guest IDTR.
6529	*/
6530	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6531	{
6532	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6533
6534	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
6535	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
6536
6537	/* Validate. */
6538	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6539
6540	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6541	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6542	}
6543
6544	return VINF_SUCCESS;
6545	}
6546
6547
6548	/**
6549	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6550	* areas.
6551	*
6552	* These MSRs will automatically be loaded to the host CPU on every successful
6553	* VM-entry and stored from the host CPU on every successful VM-exit.
6554	*
6555	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6556	* actual host MSR values are not- updated here for performance reasons. See
6557	* hmR0VmxExportHostMsrs().
6558	*
6559	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6560	*
6561	* @returns VBox status code.
6562	* @param pVCpu The cross context virtual CPU structure.
6563	* @param pVmxTransient The VMX-transient structure.
6564	*
6565	* @remarks No-long-jump zone!!!
6566	*/
6567	static int hmR0VmxExportGuestMsrs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6568	{
6569	AssertPtr(pVCpu);
6570	AssertPtr(pVmxTransient);
6571
6572	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6573	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6574
6575	/*
6576	* MSRs that we use the auto-load/store MSR area in the VMCS.
6577	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
6578	* nothing to do here. The host MSR values are updated when it's safe in
6579	* hmR0VmxLazySaveHostMsrs().
6580	*
6581	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6582	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6583	* emulation. The merged MSR permission bitmap will ensure that we get VM-exits
6584	* for any MSR that are not part of the lazy MSRs so we do not need to place
6585	* those MSRs into the auto-load/store MSR area. Nothing to do here.
6586	*/
6587	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6588	{
6589	/* No auto-load/store MSRs currently. */
6590	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6591	}
6592
6593	/*
6594	* Guest Sysenter MSRs.
6595	*/
6596	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6597	{
6598	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6599
6600	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6601	{
6602	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6603	AssertRC(rc);
6604	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6605	}
6606
6607	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6608	{
6609	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6610	AssertRC(rc);
6611	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6612	}
6613
6614	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6615	{
6616	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6617	AssertRC(rc);
6618	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6619	}
6620	}
6621
6622	/*
6623	* Guest/host EFER MSR.
6624	*/
6625	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6626	{
6627	/* Whether we are using the VMCS to swap the EFER MSR must have been
6628	determined earlier while exporting VM-entry/VM-exit controls. */
6629	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6630	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6631
6632	if (hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
6633	{
6634	/*
6635	* EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
6636	* This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
6637	* VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
6638	* and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
6639	* must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
6640	* the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
6641	* during VM-entry.
6642	*/
6643	uint64_t uGuestEferMsr = pCtx->msrEFER;
6644	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6645	{
6646	if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
6647	uGuestEferMsr &= ~MSR_K6_EFER_LME;
6648	else
6649	Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
6650	}
6651
6652	/*
6653	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6654	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6655	*/
6656	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6657	{
6658	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
6659	AssertRC(rc);
6660	}
6661	else
6662	{
6663	/*
6664	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6665	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6666	*/
6667	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
6668	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6669	AssertRCReturn(rc, rc);
6670	}
6671
6672	Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
6673	}
6674	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6675	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6676
6677	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6678	}
6679
6680	/*
6681	* Other MSRs.
6682	*/
6683	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6684	{
6685	/* Speculation Control (R/W). */
6686	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6687	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6688	{
6689	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6690	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6691	AssertRCReturn(rc, rc);
6692	}
6693
6694	/* Last Branch Record. */
6695	if (pVM->hm.s.vmx.fLbr)
6696	{
6697	uint32_t const idFromIpMsrStart = pVM->hm.s.vmx.idLbrFromIpMsrFirst;
6698	uint32_t const idToIpMsrStart = pVM->hm.s.vmx.idLbrToIpMsrFirst;
6699	uint32_t const cLbrStack = pVM->hm.s.vmx.idLbrFromIpMsrLast - pVM->hm.s.vmx.idLbrFromIpMsrFirst + 1;
6700	Assert(cLbrStack <= 32);
6701	for (uint32_t i = 0; i < cLbrStack; i++)
6702	{
6703	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idFromIpMsrStart + i,
6704	pVmxTransient->pVmcsInfo->au64LbrFromIpMsr[i],
6705	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6706	AssertRCReturn(rc, rc);
6707
6708	/* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
6709	if (idToIpMsrStart != 0)
6710	{
6711	rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idToIpMsrStart + i,
6712	pVmxTransient->pVmcsInfo->au64LbrToIpMsr[i],
6713	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6714	AssertRCReturn(rc, rc);
6715	}
6716	}
6717
6718	/* Add LBR top-of-stack MSR (which contains the index to the most recent record). */
6719	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, pVM->hm.s.vmx.idLbrTosMsr,
6720	pVmxTransient->pVmcsInfo->u64LbrTosMsr, false /* fSetReadWrite */,
6721	false /* fUpdateHostMsr */);
6722	AssertRCReturn(rc, rc);
6723	}
6724
6725	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6726	}
6727
6728	return VINF_SUCCESS;
6729	}
6730
6731
6732	/**
6733	* Wrapper for running the guest code in VT-x.
6734	*
6735	* @returns VBox status code, no informational status codes.
6736	* @param pVCpu The cross context virtual CPU structure.
6737	* @param pVmxTransient The VMX-transient structure.
6738	*
6739	* @remarks No-long-jump zone!!!
6740	*/
6741	DECLINLINE(int) hmR0VmxRunGuest(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6742	{
6743	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6744	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6745	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6746
6747	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6748
6749	/*
6750	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6751	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6752	* callee-saved and thus the need for this XMM wrapper.
6753	*
6754	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6755	*/
6756	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6757	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6758	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6759	#ifdef VBOX_WITH_KERNEL_USING_XMM
6760	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6761	#else
6762	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu);
6763	#endif
6764	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6765	return rc;
6766	}
6767
6768
6769	/**
6770	* Reports world-switch error and dumps some useful debug info.
6771	*
6772	* @param pVCpu The cross context virtual CPU structure.
6773	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6774	* @param pVmxTransient The VMX-transient structure (only
6775	* exitReason updated).
6776	*/
6777	static void hmR0VmxReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6778	{
6779	Assert(pVCpu);
6780	Assert(pVmxTransient);
6781	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6782
6783	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6784	switch (rcVMRun)
6785	{
6786	case VERR_VMX_INVALID_VMXON_PTR:
6787	AssertFailed();
6788	break;
6789	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6790	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6791	{
6792	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6793	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6794	AssertRC(rc);
6795	hmR0VmxReadExitQualVmcs(pVmxTransient);
6796
6797	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6798	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6799	Cannot do it here as we may have been long preempted. */
6800
6801	#ifdef VBOX_STRICT
6802	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6803	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6804	pVmxTransient->uExitReason));
6805	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6806	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6807	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6808	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6809	else
6810	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6811	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6812	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6813
6814	static struct
6815	{
6816	/** Name of the field to log. */
6817	const char *pszName;
6818	/** The VMCS field. */
6819	uint32_t uVmcsField;
6820	/** Whether host support of this field needs to be checked. */
6821	bool fCheckSupport;
6822	} const s_aVmcsFields[] =
6823	{
6824	{ "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false },
6825	{ "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false },
6826	{ "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true },
6827	{ "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false },
6828	{ "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false },
6829	{ "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false },
6830	{ "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false },
6831	{ "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false },
6832	{ "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false },
6833	{ "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false },
6834	{ "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false },
6835	{ "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false },
6836	{ "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false },
6837	{ "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false },
6838	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false },
6839	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false },
6840	{ "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false },
6841	{ "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false },
6842	{ "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false },
6843	{ "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false },
6844	{ "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true },
6845	{ "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false },
6846	{ "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false },
6847	{ "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false },
6848	{ "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
6849	{ "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false },
6850	{ "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false },
6851	{ "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false },
6852	/* The order of selector fields below are fixed! */
6853	{ "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false },
6854	{ "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false },
6855	{ "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false },
6856	{ "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false },
6857	{ "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false },
6858	{ "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false },
6859	{ "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false },
6860	/* End of ordered selector fields. */
6861	{ "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false },
6862	{ "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false },
6863	{ "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false },
6864	{ "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false },
6865	{ "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false },
6866	{ "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false },
6867	{ "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false },
6868	{ "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false }
6869	};
6870
6871	RTGDTR HostGdtr;
6872	ASMGetGDTR(&HostGdtr);
6873
6874	uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
6875	for (uint32_t i = 0; i < cVmcsFields; i++)
6876	{
6877	uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
6878
6879	bool fSupported;
6880	if (!s_aVmcsFields[i].fCheckSupport)
6881	fSupported = true;
6882	else
6883	{
6884	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6885	switch (uVmcsField)
6886	{
6887	case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hm.s.fNestedPaging; break;
6888	case VMX_VMCS16_VPID: fSupported = pVM->hm.s.vmx.fVpid; break;
6889	case VMX_VMCS32_CTRL_PROC_EXEC2:
6890	fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6891	break;
6892	default:
6893	AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
6894	}
6895	}
6896
6897	if (fSupported)
6898	{
6899	uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
6900	switch (uWidth)
6901	{
6902	case VMX_VMCSFIELD_WIDTH_16BIT:
6903	{
6904	uint16_t u16Val;
6905	rc = VMXReadVmcs16(uVmcsField, &u16Val);
6906	AssertRC(rc);
6907	Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
6908
6909	if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
6910	&& uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
6911	{
6912	if (u16Val < HostGdtr.cbGdt)
6913	{
6914	/* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
6915	static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
6916	"Host FS", "Host GS", "Host TR" };
6917	uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
6918	Assert(idxSel < RT_ELEMENTS(s_apszSel));
6919	PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
6920	hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
6921	}
6922	else
6923	Log4((" Selector value exceeds GDT limit!\n"));
6924	}
6925	break;
6926	}
6927
6928	case VMX_VMCSFIELD_WIDTH_32BIT:
6929	{
6930	uint32_t u32Val;
6931	rc = VMXReadVmcs32(uVmcsField, &u32Val);
6932	AssertRC(rc);
6933	Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
6934	break;
6935	}
6936
6937	case VMX_VMCSFIELD_WIDTH_64BIT:
6938	case VMX_VMCSFIELD_WIDTH_NATURAL:
6939	{
6940	uint64_t u64Val;
6941	rc = VMXReadVmcs64(uVmcsField, &u64Val);
6942	AssertRC(rc);
6943	Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
6944	break;
6945	}
6946	}
6947	}
6948	}
6949
6950	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6951	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6952	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6953	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6954	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6955	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6956	#endif /* VBOX_STRICT */
6957	break;
6958	}
6959
6960	default:
6961	/* Impossible */
6962	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6963	break;
6964	}
6965	}
6966
6967
6968	/**
6969	* Sets up the usage of TSC-offsetting and updates the VMCS.
6970	*
6971	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6972	* VMX-preemption timer.
6973	*
6974	* @returns VBox status code.
6975	* @param pVCpu The cross context virtual CPU structure.
6976	* @param pVmxTransient The VMX-transient structure.
6977	*
6978	* @remarks No-long-jump zone!!!
6979	*/
6980	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6981	{
6982	bool fOffsettedTsc;
6983	bool fParavirtTsc;
6984	uint64_t uTscOffset;
6985	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6986	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6987
6988	if (pVM->hm.s.vmx.fUsePreemptTimer)
6989	{
6990	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6991
6992	/* Make sure the returned values have sane upper and lower boundaries. */
6993	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6994	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6995	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6996	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6997
6998	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6999	* preemption timers here. We probably need to clamp the preemption timer,
7000	* after converting the timer value to the host. */
7001	uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7002	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7003	AssertRC(rc);
7004	}
7005	else
7006	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7007
7008	if (fParavirtTsc)
7009	{
7010	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7011	information before every VM-entry, hence disable it for performance sake. */
7012	#if 0
7013	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7014	AssertRC(rc);
7015	#endif
7016	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7017	}
7018
7019	if ( fOffsettedTsc
7020	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7021	{
7022	if (pVmxTransient->fIsNestedGuest)
7023	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7024	hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
7025	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7026	}
7027	else
7028	{
7029	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7030	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7031	}
7032	}
7033
7034
7035	/**
7036	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7037	* VM-exit interruption info type.
7038	*
7039	* @returns The IEM exception flags.
7040	* @param uVector The event vector.
7041	* @param uVmxEventType The VMX event type.
7042	*
7043	* @remarks This function currently only constructs flags required for
7044	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7045	* and CR2 aspects of an exception are not included).
7046	*/
7047	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7048	{
7049	uint32_t fIemXcptFlags;
7050	switch (uVmxEventType)
7051	{
7052	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7053	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7054	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7055	break;
7056
7057	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7058	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7059	break;
7060
7061	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7062	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7063	break;
7064
7065	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7066	{
7067	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7068	if (uVector == X86_XCPT_BP)
7069	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7070	else if (uVector == X86_XCPT_OF)
7071	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7072	else
7073	{
7074	fIemXcptFlags = 0;
7075	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7076	}
7077	break;
7078	}
7079
7080	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7081	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7082	break;
7083
7084	default:
7085	fIemXcptFlags = 0;
7086	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7087	break;
7088	}
7089	return fIemXcptFlags;
7090	}
7091
7092
7093	/**
7094	* Sets an event as a pending event to be injected into the guest.
7095	*
7096	* @param pVCpu The cross context virtual CPU structure.
7097	* @param u32IntInfo The VM-entry interruption-information field.
7098	* @param cbInstr The VM-entry instruction length in bytes (for
7099	* software interrupts, exceptions and privileged
7100	* software exceptions).
7101	* @param u32ErrCode The VM-entry exception error code.
7102	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7103	* page-fault.
7104	*/
7105	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7106	RTGCUINTPTR GCPtrFaultAddress)
7107	{
7108	Assert(!pVCpu->hm.s.Event.fPending);
7109	pVCpu->hm.s.Event.fPending = true;
7110	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7111	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7112	pVCpu->hm.s.Event.cbInstr = cbInstr;
7113	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7114	}
7115
7116
7117	/**
7118	* Sets an external interrupt as pending-for-injection into the VM.
7119	*
7120	* @param pVCpu The cross context virtual CPU structure.
7121	* @param u8Interrupt The external interrupt vector.
7122	*/
7123	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
7124	{
7125	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7126	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7127	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7128	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7129	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7130	}
7131
7132
7133	/**
7134	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7135	*
7136	* @param pVCpu The cross context virtual CPU structure.
7137	*/
7138	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPUCC pVCpu)
7139	{
7140	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7141	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7142	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7143	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7144	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7145	}
7146
7147
7148	/**
7149	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7150	*
7151	* @param pVCpu The cross context virtual CPU structure.
7152	*/
7153	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPUCC pVCpu)
7154	{
7155	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7156	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7157	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7158	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7159	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7160	}
7161
7162
7163	/**
7164	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7165	*
7166	* @param pVCpu The cross context virtual CPU structure.
7167	*/
7168	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPUCC pVCpu)
7169	{
7170	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7171	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7172	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7173	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7174	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7175	}
7176
7177
7178	/**
7179	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7180	*
7181	* @param pVCpu The cross context virtual CPU structure.
7182	*/
7183	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPUCC pVCpu)
7184	{
7185	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7186	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7187	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7188	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7189	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7190	}
7191
7192
7193	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7194	/**
7195	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7196	*
7197	* @param pVCpu The cross context virtual CPU structure.
7198	* @param u32ErrCode The error code for the general-protection exception.
7199	*/
7200	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7201	{
7202	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7203	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7204	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7205	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7206	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7207	}
7208
7209
7210	/**
7211	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7212	*
7213	* @param pVCpu The cross context virtual CPU structure.
7214	* @param u32ErrCode The error code for the stack exception.
7215	*/
7216	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7217	{
7218	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7219	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7220	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7221	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7222	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7223	}
7224	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7225
7226
7227	/**
7228	* Fixes up attributes for the specified segment register.
7229	*
7230	* @param pVCpu The cross context virtual CPU structure.
7231	* @param pSelReg The segment register that needs fixing.
7232	* @param pszRegName The register name (for logging and assertions).
7233	*/
7234	static void hmR0VmxFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
7235	{
7236	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7237
7238	/*
7239	* If VT-x marks the segment as unusable, most other bits remain undefined:
7240	* - For CS the L, D and G bits have meaning.
7241	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7242	* - For the remaining data segments no bits are defined.
7243	*
7244	* The present bit and the unusable bit has been observed to be set at the
7245	* same time (the selector was supposed to be invalid as we started executing
7246	* a V8086 interrupt in ring-0).
7247	*
7248	* What should be important for the rest of the VBox code, is that the P bit is
7249	* cleared. Some of the other VBox code recognizes the unusable bit, but
7250	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7251	* safe side here, we'll strip off P and other bits we don't care about. If
7252	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7253	*
7254	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7255	*/
7256	#ifdef VBOX_STRICT
7257	uint32_t const uAttr = pSelReg->Attr.u;
7258	#endif
7259
7260	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7261	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7262	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7263
7264	#ifdef VBOX_STRICT
7265	VMMRZCallRing3Disable(pVCpu);
7266	Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7267	# ifdef DEBUG_bird
7268	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7269	("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7270	pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7271	# endif
7272	VMMRZCallRing3Enable(pVCpu);
7273	NOREF(uAttr);
7274	#endif
7275	RT_NOREF2(pVCpu, pszRegName);
7276	}
7277
7278
7279	/**
7280	* Imports a guest segment register from the current VMCS into the guest-CPU
7281	* context.
7282	*
7283	* @param pVCpu The cross context virtual CPU structure.
7284	* @param iSegReg The segment register number (X86_SREG_XXX).
7285	*
7286	* @remarks Called with interrupts and/or preemption disabled.
7287	*/
7288	static void hmR0VmxImportGuestSegReg(PVMCPUCC pVCpu, uint32_t iSegReg)
7289	{
7290	Assert(iSegReg < X86_SREG_COUNT);
7291	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
7292	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
7293	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
7294	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
7295
7296	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7297
7298	uint16_t u16Sel;
7299	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_SEG_SEL(iSegReg), &u16Sel); AssertRC(rc);
7300	pSelReg->Sel = u16Sel;
7301	pSelReg->ValidSel = u16Sel;
7302
7303	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), &pSelReg->u32Limit); AssertRC(rc);
7304	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SEG_BASE(iSegReg), &pSelReg->u64Base); AssertRC(rc);
7305
7306	uint32_t u32Attr;
7307	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), &u32Attr); AssertRC(rc);
7308	pSelReg->Attr.u = u32Attr;
7309	if (u32Attr & X86DESCATTR_UNUSABLE)
7310	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + iSegReg * 3);
7311
7312	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7313	}
7314
7315
7316	/**
7317	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7318	*
7319	* @param pVCpu The cross context virtual CPU structure.
7320	*
7321	* @remarks Called with interrupts and/or preemption disabled.
7322	*/
7323	static void hmR0VmxImportGuestLdtr(PVMCPUCC pVCpu)
7324	{
7325	uint16_t u16Sel;
7326	uint64_t u64Base;
7327	uint32_t u32Limit, u32Attr;
7328	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
7329	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
7330	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7331	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
7332
7333	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
7334	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
7335	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7336	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7337	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7338	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7339	if (u32Attr & X86DESCATTR_UNUSABLE)
7340	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
7341	}
7342
7343
7344	/**
7345	* Imports the guest TR from the current VMCS into the guest-CPU context.
7346	*
7347	* @param pVCpu The cross context virtual CPU structure.
7348	*
7349	* @remarks Called with interrupts and/or preemption disabled.
7350	*/
7351	static void hmR0VmxImportGuestTr(PVMCPUCC pVCpu)
7352	{
7353	uint16_t u16Sel;
7354	uint64_t u64Base;
7355	uint32_t u32Limit, u32Attr;
7356	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
7357	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
7358	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7359	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
7360
7361	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
7362	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
7363	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7364	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7365	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7366	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7367	/* TR is the only selector that can never be unusable. */
7368	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7369	}
7370
7371
7372	/**
7373	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7374	*
7375	* @param pVCpu The cross context virtual CPU structure.
7376	*
7377	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7378	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7379	* instead!!!
7380	*/
7381	static void hmR0VmxImportGuestRip(PVMCPUCC pVCpu)
7382	{
7383	uint64_t u64Val;
7384	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7385	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7386	{
7387	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
7388	AssertRC(rc);
7389
7390	pCtx->rip = u64Val;
7391	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7392	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7393	}
7394	}
7395
7396
7397	/**
7398	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7399	*
7400	* @param pVCpu The cross context virtual CPU structure.
7401	* @param pVmcsInfo The VMCS info. object.
7402	*
7403	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7404	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7405	* instead!!!
7406	*/
7407	static void hmR0VmxImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7408	{
7409	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7410	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7411	{
7412	uint64_t u64Val;
7413	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7414	AssertRC(rc);
7415
7416	pCtx->rflags.u64 = u64Val;
7417	if (pVmcsInfo->RealMode.fRealOnV86Active)
7418	{
7419	pCtx->eflags.Bits.u1VM = 0;
7420	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7421	}
7422	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7423	}
7424	}
7425
7426
7427	/**
7428	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7429	* context.
7430	*
7431	* @param pVCpu The cross context virtual CPU structure.
7432	* @param pVmcsInfo The VMCS info. object.
7433	*
7434	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7435	* do not log!
7436	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7437	* instead!!!
7438	*/
7439	static void hmR0VmxImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7440	{
7441	uint32_t u32Val;
7442	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
7443	if (!u32Val)
7444	{
7445	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7446	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7447	CPUMSetGuestNmiBlocking(pVCpu, false);
7448	}
7449	else
7450	{
7451	/*
7452	* We must import RIP here to set our EM interrupt-inhibited state.
7453	* We also import RFLAGS as our code that evaluates pending interrupts
7454	* before VM-entry requires it.
7455	*/
7456	hmR0VmxImportGuestRip(pVCpu);
7457	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7458
7459	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7460	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7461	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7462	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7463
7464	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7465	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7466	}
7467	}
7468
7469
7470	/**
7471	* Worker for VMXR0ImportStateOnDemand.
7472	*
7473	* @returns VBox status code.
7474	* @param pVCpu The cross context virtual CPU structure.
7475	* @param pVmcsInfo The VMCS info. object.
7476	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7477	*/
7478	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7479	{
7480	int rc = VINF_SUCCESS;
7481	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7482	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7483	uint32_t u32Val;
7484
7485	/*
7486	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7487	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7488	* neither are other host platforms.
7489	*
7490	* Committing this temporarily as it prevents BSOD.
7491	*
7492	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7493	*/
7494	#ifdef RT_OS_WINDOWS
7495	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7496	return VERR_HM_IPE_1;
7497	#endif
7498
7499	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7500
7501	/*
7502	* We disable interrupts to make the updating of the state and in particular
7503	* the fExtrn modification atomic wrt to preemption hooks.
7504	*/
7505	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7506
7507	fWhat &= pCtx->fExtrn;
7508	if (fWhat)
7509	{
7510	do
7511	{
7512	if (fWhat & CPUMCTX_EXTRN_RIP)
7513	hmR0VmxImportGuestRip(pVCpu);
7514
7515	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7516	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7517
7518	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7519	hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7520
7521	if (fWhat & CPUMCTX_EXTRN_RSP)
7522	{
7523	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &pCtx->rsp);
7524	AssertRC(rc);
7525	}
7526
7527	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7528	{
7529	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7530	if (fWhat & CPUMCTX_EXTRN_CS)
7531	{
7532	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7533	hmR0VmxImportGuestRip(pVCpu);
7534	if (fRealOnV86Active)
7535	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7536	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7537	}
7538	if (fWhat & CPUMCTX_EXTRN_SS)
7539	{
7540	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7541	if (fRealOnV86Active)
7542	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7543	}
7544	if (fWhat & CPUMCTX_EXTRN_DS)
7545	{
7546	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7547	if (fRealOnV86Active)
7548	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7549	}
7550	if (fWhat & CPUMCTX_EXTRN_ES)
7551	{
7552	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7553	if (fRealOnV86Active)
7554	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7555	}
7556	if (fWhat & CPUMCTX_EXTRN_FS)
7557	{
7558	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7559	if (fRealOnV86Active)
7560	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7561	}
7562	if (fWhat & CPUMCTX_EXTRN_GS)
7563	{
7564	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7565	if (fRealOnV86Active)
7566	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7567	}
7568	}
7569
7570	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7571	{
7572	if (fWhat & CPUMCTX_EXTRN_LDTR)
7573	hmR0VmxImportGuestLdtr(pVCpu);
7574
7575	if (fWhat & CPUMCTX_EXTRN_GDTR)
7576	{
7577	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
7578	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
7579	pCtx->gdtr.cbGdt = u32Val;
7580	}
7581
7582	/* Guest IDTR. */
7583	if (fWhat & CPUMCTX_EXTRN_IDTR)
7584	{
7585	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
7586	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
7587	pCtx->idtr.cbIdt = u32Val;
7588	}
7589
7590	/* Guest TR. */
7591	if (fWhat & CPUMCTX_EXTRN_TR)
7592	{
7593	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7594	don't need to import that one. */
7595	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7596	hmR0VmxImportGuestTr(pVCpu);
7597	}
7598	}
7599
7600	if (fWhat & CPUMCTX_EXTRN_DR7)
7601	{
7602	if (!pVCpu->hm.s.fUsingHyperDR7)
7603	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &pCtx->dr[7]); AssertRC(rc);
7604	}
7605
7606	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7607	{
7608	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
7609	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
7610	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
7611	pCtx->SysEnter.cs = u32Val;
7612	}
7613
7614	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7615	{
7616	if ( pVM->hm.s.fAllow64BitGuests
7617	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7618	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7619	}
7620
7621	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7622	{
7623	if ( pVM->hm.s.fAllow64BitGuests
7624	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7625	{
7626	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7627	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7628	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7629	}
7630	}
7631
7632	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7633	{
7634	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7635	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7636	Assert(pMsrs);
7637	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7638	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7639	for (uint32_t i = 0; i < cMsrs; i++)
7640	{
7641	uint32_t const idMsr = pMsrs[i].u32Msr;
7642	switch (idMsr)
7643	{
7644	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7645	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7646	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7647	default:
7648	{
7649	uint32_t idxLbrMsr;
7650	if (pVM->hm.s.vmx.fLbr)
7651	{
7652	if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
7653	{
7654	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfo->au64LbrFromIpMsr));
7655	pVmcsInfo->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
7656	break;
7657	}
7658	else if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
7659	{
7660	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfo->au64LbrFromIpMsr));
7661	pVmcsInfo->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
7662	break;
7663	}
7664	else if (idMsr == pVM->hm.s.vmx.idLbrTosMsr)
7665	{
7666	pVmcsInfo->u64LbrTosMsr = pMsrs[i].u64Value;
7667	break;
7668	}
7669	/* Fallthru (no break) */
7670	}
7671	pCtx->fExtrn = 0;
7672	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7673	ASMSetFlags(fEFlags);
7674	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7675	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7676	}
7677	}
7678	}
7679	}
7680
7681	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7682	{
7683	if (fWhat & CPUMCTX_EXTRN_CR0)
7684	{
7685	uint64_t u64Cr0;
7686	uint64_t u64Shadow;
7687	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
7688	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
7689	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7690	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7691	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7692	#else
7693	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7694	{
7695	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7696	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7697	}
7698	else
7699	{
7700	/*
7701	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
7702	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7703	* re-construct CR0. See @bugref{9180#c95} for details.
7704	*/
7705	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7706	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7707	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7708	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
7709	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
7710	}
7711	#endif
7712	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7713	CPUMSetGuestCR0(pVCpu, u64Cr0);
7714	VMMRZCallRing3Enable(pVCpu);
7715	}
7716
7717	if (fWhat & CPUMCTX_EXTRN_CR4)
7718	{
7719	uint64_t u64Cr4;
7720	uint64_t u64Shadow;
7721	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
7722	rc \|= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
7723	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7724	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7725	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7726	#else
7727	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7728	{
7729	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7730	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7731	}
7732	else
7733	{
7734	/*
7735	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
7736	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7737	* re-construct CR4. See @bugref{9180#c95} for details.
7738	*/
7739	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7740	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7741	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7742	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
7743	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
7744	}
7745	#endif
7746	pCtx->cr4 = u64Cr4;
7747	}
7748
7749	if (fWhat & CPUMCTX_EXTRN_CR3)
7750	{
7751	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7752	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7753	\|\| ( pVM->hm.s.fNestedPaging
7754	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7755	{
7756	uint64_t u64Cr3;
7757	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
7758	if (pCtx->cr3 != u64Cr3)
7759	{
7760	pCtx->cr3 = u64Cr3;
7761	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7762	}
7763
7764	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7765	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7766	if (CPUMIsGuestInPAEModeEx(pCtx))
7767	{
7768	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
7769	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
7770	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
7771	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
7772	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7773	}
7774	}
7775	}
7776	}
7777
7778	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7779	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7780	{
7781	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7782	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7783	{
7784	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7785	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7786	if (RT_SUCCESS(rc))
7787	{ /* likely */ }
7788	else
7789	break;
7790	}
7791	}
7792	#endif
7793	} while (0);
7794
7795	if (RT_SUCCESS(rc))
7796	{
7797	/* Update fExtrn. */
7798	pCtx->fExtrn &= ~fWhat;
7799
7800	/* If everything has been imported, clear the HM keeper bit. */
7801	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7802	{
7803	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7804	Assert(!pCtx->fExtrn);
7805	}
7806	}
7807	}
7808	else
7809	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7810
7811	/*
7812	* Restore interrupts.
7813	*/
7814	ASMSetFlags(fEFlags);
7815
7816	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7817
7818	if (RT_SUCCESS(rc))
7819	{ /* likely */ }
7820	else
7821	return rc;
7822
7823	/*
7824	* Honor any pending CR3 updates.
7825	*
7826	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
7827	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7828	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7829	*
7830	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7831	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7832	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7833	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7834	*
7835	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7836	*/
7837	if (VMMRZCallRing3IsEnabled(pVCpu))
7838	{
7839	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7840	{
7841	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7842	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7843	}
7844
7845	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7846	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7847
7848	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7849	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7850	}
7851
7852	return VINF_SUCCESS;
7853	}
7854
7855
7856	/**
7857	* Saves the guest state from the VMCS into the guest-CPU context.
7858	*
7859	* @returns VBox status code.
7860	* @param pVCpu The cross context virtual CPU structure.
7861	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7862	*/
7863	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
7864	{
7865	AssertPtr(pVCpu);
7866	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7867	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7868	}
7869
7870
7871	/**
7872	* Check per-VM and per-VCPU force flag actions that require us to go back to
7873	* ring-3 for one reason or another.
7874	*
7875	* @returns Strict VBox status code (i.e. informational status codes too)
7876	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7877	* ring-3.
7878	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7879	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7880	* interrupts)
7881	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7882	* all EMTs to be in ring-3.
7883	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7884	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7885	* to the EM loop.
7886	*
7887	* @param pVCpu The cross context virtual CPU structure.
7888	* @param pVmxTransient The VMX-transient structure.
7889	* @param fStepping Whether we are single-stepping the guest using the
7890	* hypervisor debugger.
7891	*
7892	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7893	* is no longer in VMX non-root mode.
7894	*/
7895	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, bool fStepping)
7896	{
7897	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7898
7899	/*
7900	* Update pending interrupts into the APIC's IRR.
7901	*/
7902	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7903	APICUpdatePendingInterrupts(pVCpu);
7904
7905	/*
7906	* Anything pending? Should be more likely than not if we're doing a good job.
7907	*/
7908	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7909	if ( !fStepping
7910	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7911	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7912	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7913	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7914	return VINF_SUCCESS;
7915
7916	/* Pending PGM C3 sync. */
7917	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7918	{
7919	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7920	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7921	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7922	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7923	if (rcStrict != VINF_SUCCESS)
7924	{
7925	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7926	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7927	return rcStrict;
7928	}
7929	}
7930
7931	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7932	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7933	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7934	{
7935	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7936	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7937	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7938	return rc;
7939	}
7940
7941	/* Pending VM request packets, such as hardware interrupts. */
7942	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7943	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7944	{
7945	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7946	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7947	return VINF_EM_PENDING_REQUEST;
7948	}
7949
7950	/* Pending PGM pool flushes. */
7951	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7952	{
7953	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7954	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7955	return VINF_PGM_POOL_FLUSH_PENDING;
7956	}
7957
7958	/* Pending DMA requests. */
7959	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7960	{
7961	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7962	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7963	return VINF_EM_RAW_TO_R3;
7964	}
7965
7966	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7967	/*
7968	* Pending nested-guest events.
7969	*
7970	* Please note the priority of these events are specified and important.
7971	* See Intel spec. 29.4.3.2 "APIC-Write Emulation".
7972	* See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
7973	*/
7974	if (pVmxTransient->fIsNestedGuest)
7975	{
7976	/* Pending nested-guest APIC-write. */
7977	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7978	{
7979	Log4Func(("Pending nested-guest APIC-write\n"));
7980	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
7981	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7982	return rcStrict;
7983	}
7984
7985	/* Pending nested-guest monitor-trap flag (MTF). */
7986	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
7987	{
7988	Log4Func(("Pending nested-guest MTF\n"));
7989	VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
7990	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7991	return rcStrict;
7992	}
7993
7994	/* Pending nested-guest VMX-preemption timer expired. */
7995	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
7996	{
7997	Log4Func(("Pending nested-guest MTF\n"));
7998	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
7999	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8000	return rcStrict;
8001	}
8002	}
8003	#else
8004	NOREF(pVmxTransient);
8005	#endif
8006
8007	return VINF_SUCCESS;
8008	}
8009
8010
8011	/**
8012	* Converts any TRPM trap into a pending HM event. This is typically used when
8013	* entering from ring-3 (not longjmp returns).
8014	*
8015	* @param pVCpu The cross context virtual CPU structure.
8016	*/
8017	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
8018	{
8019	Assert(TRPMHasTrap(pVCpu));
8020	Assert(!pVCpu->hm.s.Event.fPending);
8021
8022	uint8_t uVector;
8023	TRPMEVENT enmTrpmEvent;
8024	uint32_t uErrCode;
8025	RTGCUINTPTR GCPtrFaultAddress;
8026	uint8_t cbInstr;
8027	bool fIcebp;
8028
8029	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
8030	AssertRC(rc);
8031
8032	uint32_t u32IntInfo;
8033	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
8034	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
8035
8036	rc = TRPMResetTrap(pVCpu);
8037	AssertRC(rc);
8038	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8039	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8040
8041	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8042	}
8043
8044
8045	/**
8046	* Converts the pending HM event into a TRPM trap.
8047	*
8048	* @param pVCpu The cross context virtual CPU structure.
8049	*/
8050	static void hmR0VmxPendingEventToTrpmTrap(PVMCPUCC pVCpu)
8051	{
8052	Assert(pVCpu->hm.s.Event.fPending);
8053
8054	/* If a trap was already pending, we did something wrong! */
8055	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8056
8057	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
8058	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
8059	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
8060
8061	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8062
8063	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8064	AssertRC(rc);
8065
8066	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8067	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
8068
8069	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
8070	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8071	else
8072	{
8073	uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
8074	switch (uVectorType)
8075	{
8076	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
8077	TRPMSetTrapDueToIcebp(pVCpu);
8078	RT_FALL_THRU();
8079	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8080	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
8081	{
8082	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8083	\|\| ( uVector == X86_XCPT_BP /* INT3 */
8084	\|\| uVector == X86_XCPT_OF /* INTO */
8085	\|\| uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
8086	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
8087	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8088	break;
8089	}
8090	}
8091	}
8092
8093	/* We're now done converting the pending event. */
8094	pVCpu->hm.s.Event.fPending = false;
8095	}
8096
8097
8098	/**
8099	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8100	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8101	*
8102	* @param pVCpu The cross context virtual CPU structure.
8103	* @param pVmcsInfo The VMCS info. object.
8104	*/
8105	static void hmR0VmxSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
8106	{
8107	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8108	{
8109	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8110	{
8111	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8112	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8113	AssertRC(rc);
8114	}
8115	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8116	}
8117
8118
8119	/**
8120	* Clears the interrupt-window exiting control in the VMCS.
8121	*
8122	* @param pVmcsInfo The VMCS info. object.
8123	*/
8124	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8125	{
8126	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8127	{
8128	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8129	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8130	AssertRC(rc);
8131	}
8132	}
8133
8134
8135	/**
8136	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8137	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8138	*
8139	* @param pVCpu The cross context virtual CPU structure.
8140	* @param pVmcsInfo The VMCS info. object.
8141	*/
8142	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
8143	{
8144	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8145	{
8146	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8147	{
8148	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8149	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8150	AssertRC(rc);
8151	Log4Func(("Setup NMI-window exiting\n"));
8152	}
8153	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8154	}
8155
8156
8157	/**
8158	* Clears the NMI-window exiting control in the VMCS.
8159	*
8160	* @param pVmcsInfo The VMCS info. object.
8161	*/
8162	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8163	{
8164	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8165	{
8166	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8167	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8168	AssertRC(rc);
8169	}
8170	}
8171
8172
8173	/**
8174	* Does the necessary state syncing before returning to ring-3 for any reason
8175	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8176	*
8177	* @returns VBox status code.
8178	* @param pVCpu The cross context virtual CPU structure.
8179	* @param fImportState Whether to import the guest state from the VMCS back
8180	* to the guest-CPU context.
8181	*
8182	* @remarks No-long-jmp zone!!!
8183	*/
8184	static int hmR0VmxLeave(PVMCPUCC pVCpu, bool fImportState)
8185	{
8186	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8187	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8188
8189	RTCPUID const idCpu = RTMpCpuId();
8190	Log4Func(("HostCpuId=%u\n", idCpu));
8191
8192	/*
8193	* !!! IMPORTANT !!!
8194	* If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
8195	*/
8196
8197	/* Save the guest state if necessary. */
8198	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8199	if (fImportState)
8200	{
8201	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8202	AssertRCReturn(rc, rc);
8203	}
8204
8205	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8206	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8207	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8208
8209	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8210	#ifdef VBOX_STRICT
8211	if (CPUMIsHyperDebugStateActive(pVCpu))
8212	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8213	#endif
8214	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8215	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
8216	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
8217
8218	/* Restore host-state bits that VT-x only restores partially. */
8219	if (pVCpu->hm.s.vmx.fRestoreHostFlags > VMX_RESTORE_HOST_REQUIRED)
8220	{
8221	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8222	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8223	}
8224	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8225
8226	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8227	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8228	{
8229	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8230	if (!fImportState)
8231	{
8232	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8233	AssertRCReturn(rc, rc);
8234	}
8235	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8236	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8237	}
8238	else
8239	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8240
8241	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8242	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8243
8244	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8245	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8246	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8247	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8248	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8249	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8250	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8251	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8252	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8253	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8254
8255	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8256
8257	/** @todo This partially defeats the purpose of having preemption hooks.
8258	* The problem is, deregistering the hooks should be moved to a place that
8259	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8260	* context.
8261	*/
8262	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8263	AssertRCReturn(rc, rc);
8264
8265	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8266	/*
8267	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8268	* clear a shadow VMCS before allowing that VMCS to become active on another
8269	* logical processor. We may or may not be importing guest state which clears
8270	* it, so cover for it here.
8271	*
8272	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8273	*/
8274	if ( pVmcsInfo->pvShadowVmcs
8275	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8276	{
8277	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8278	AssertRCReturn(rc, rc);
8279	}
8280
8281	/*
8282	* Flag that we need to re-export the host state if we switch to this VMCS before
8283	* executing guest or nested-guest code.
8284	*/
8285	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
8286	#endif
8287
8288	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8289	NOREF(idCpu);
8290	return VINF_SUCCESS;
8291	}
8292
8293
8294	/**
8295	* Leaves the VT-x session.
8296	*
8297	* @returns VBox status code.
8298	* @param pVCpu The cross context virtual CPU structure.
8299	*
8300	* @remarks No-long-jmp zone!!!
8301	*/
8302	static int hmR0VmxLeaveSession(PVMCPUCC pVCpu)
8303	{
8304	HM_DISABLE_PREEMPT(pVCpu);
8305	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8306	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8307	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8308
8309	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8310	and done this from the VMXR0ThreadCtxCallback(). */
8311	if (!pVCpu->hm.s.fLeaveDone)
8312	{
8313	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8314	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8315	pVCpu->hm.s.fLeaveDone = true;
8316	}
8317	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8318
8319	/*
8320	* !!! IMPORTANT !!!
8321	* If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
8322	*/
8323
8324	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8325	/** @todo Deregistering here means we need to VMCLEAR always
8326	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8327	* for calling VMMR0ThreadCtxHookDisable here! */
8328	VMMR0ThreadCtxHookDisable(pVCpu);
8329
8330	/* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
8331	int rc = HMR0LeaveCpu(pVCpu);
8332	HM_RESTORE_PREEMPT();
8333	return rc;
8334	}
8335
8336
8337	/**
8338	* Does the necessary state syncing before doing a longjmp to ring-3.
8339	*
8340	* @returns VBox status code.
8341	* @param pVCpu The cross context virtual CPU structure.
8342	*
8343	* @remarks No-long-jmp zone!!!
8344	*/
8345	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPUCC pVCpu)
8346	{
8347	return hmR0VmxLeaveSession(pVCpu);
8348	}
8349
8350
8351	/**
8352	* Take necessary actions before going back to ring-3.
8353	*
8354	* An action requires us to go back to ring-3. This function does the necessary
8355	* steps before we can safely return to ring-3. This is not the same as longjmps
8356	* to ring-3, this is voluntary and prepares the guest so it may continue
8357	* executing outside HM (recompiler/IEM).
8358	*
8359	* @returns VBox status code.
8360	* @param pVCpu The cross context virtual CPU structure.
8361	* @param rcExit The reason for exiting to ring-3. Can be
8362	* VINF_VMM_UNKNOWN_RING3_CALL.
8363	*/
8364	static int hmR0VmxExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
8365	{
8366	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8367
8368	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8369	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8370	{
8371	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8372	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8373	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8374	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8375	}
8376
8377	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8378	VMMRZCallRing3Disable(pVCpu);
8379	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8380
8381	/*
8382	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8383	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8384	*
8385	* This is because execution may continue from ring-3 and we would need to inject
8386	* the event from there (hence place it back in TRPM).
8387	*/
8388	if (pVCpu->hm.s.Event.fPending)
8389	{
8390	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8391	Assert(!pVCpu->hm.s.Event.fPending);
8392
8393	/* Clear the events from the VMCS. */
8394	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
8395	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
8396	}
8397	#ifdef VBOX_STRICT
8398	/*
8399	* We check for rcExit here since for errors like VERR_VMX_UNABLE_TO_START_VM (which are
8400	* fatal), we don't care about verifying duplicate injection of events. Errors like
8401	* VERR_EM_INTERPRET are converted to their VINF_* counterparts -prior- to calling this
8402	* function so those should and will be checked below.
8403	*/
8404	else if (RT_SUCCESS(rcExit))
8405	{
8406	/*
8407	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8408	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8409	* occasionally, see @bugref{9180#c42}.
8410	*
8411	* However, if the VM-entry failed, any VM entry-interruption info. field would
8412	* be left unmodified as the event would not have been injected to the guest. In
8413	* such cases, don't assert, we're not going to continue guest execution anyway.
8414	*/
8415	uint32_t uExitReason;
8416	uint32_t uEntryIntInfo;
8417	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8418	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8419	AssertRC(rc);
8420	AssertMsg(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo),
8421	("uExitReason=%#RX32 uEntryIntInfo=%#RX32 rcExit=%d\n", uExitReason, uEntryIntInfo, VBOXSTRICTRC_VAL(rcExit)));
8422	}
8423	#endif
8424
8425	/*
8426	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8427	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8428	* (e.g. TPR below threshold).
8429	*/
8430	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8431	{
8432	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8433	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8434	}
8435
8436	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8437	and if we're injecting an event we should have a TRPM trap pending. */
8438	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8439	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8440	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8441	#endif
8442
8443	/* Save guest state and restore host state bits. */
8444	int rc = hmR0VmxLeaveSession(pVCpu);
8445	AssertRCReturn(rc, rc);
8446	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8447
8448	/* Thread-context hooks are unregistered at this point!!! */
8449	/* Ring-3 callback notifications are unregistered at this point!!! */
8450
8451	/* Sync recompiler state. */
8452	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8453	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8454	\| CPUM_CHANGED_LDTR
8455	\| CPUM_CHANGED_GDTR
8456	\| CPUM_CHANGED_IDTR
8457	\| CPUM_CHANGED_TR
8458	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8459	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8460	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8461	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8462
8463	Assert(!pVCpu->hm.s.fClearTrapFlag);
8464
8465	/* Update the exit-to-ring 3 reason. */
8466	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8467
8468	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8469	if ( rcExit != VINF_EM_RAW_INTERRUPT
8470	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8471	{
8472	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8474	}
8475
8476	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8477	VMMRZCallRing3Enable(pVCpu);
8478	return rc;
8479	}
8480
8481
8482	/**
8483	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8484	* longjump to ring-3 and possibly get preempted.
8485	*
8486	* @returns VBox status code.
8487	* @param pVCpu The cross context virtual CPU structure.
8488	* @param enmOperation The operation causing the ring-3 longjump.
8489	*/
8490	VMMR0DECL(int) VMXR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
8491	{
8492	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8493	{
8494	/*
8495	* !!! IMPORTANT !!!
8496	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8497	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8498	*/
8499	VMMRZCallRing3RemoveNotification(pVCpu);
8500	VMMRZCallRing3Disable(pVCpu);
8501	HM_DISABLE_PREEMPT(pVCpu);
8502
8503	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8504	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8505	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8506	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8507
8508	/* Restore host-state bits that VT-x only restores partially. */
8509	if (pVCpu->hm.s.vmx.fRestoreHostFlags > VMX_RESTORE_HOST_REQUIRED)
8510	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8511	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8512
8513	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8514	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8515	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8516
8517	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8518	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8519	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8520
8521	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8522	cleared as part of importing the guest state above. */
8523	hmR0VmxClearVmcs(pVmcsInfo);
8524
8525	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8526	VMMR0ThreadCtxHookDisable(pVCpu);
8527
8528	/* Leave HM context. This takes care of local init (term). */
8529	HMR0LeaveCpu(pVCpu);
8530	HM_RESTORE_PREEMPT();
8531	return VINF_SUCCESS;
8532	}
8533
8534	Assert(pVCpu);
8535	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8536	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8537
8538	VMMRZCallRing3Disable(pVCpu);
8539	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8540
8541	Log4Func(("-> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8542
8543	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8544	AssertRCReturn(rc, rc);
8545
8546	VMMRZCallRing3Enable(pVCpu);
8547	return VINF_SUCCESS;
8548	}
8549
8550
8551	/**
8552	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8553	* stack.
8554	*
8555	* @returns Strict VBox status code (i.e. informational status codes too).
8556	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8557	* @param pVCpu The cross context virtual CPU structure.
8558	* @param uValue The value to push to the guest stack.
8559	*/
8560	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
8561	{
8562	/*
8563	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8564	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8565	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8566	*/
8567	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8568	if (pCtx->sp == 1)
8569	return VINF_EM_RESET;
8570	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8571	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8572	AssertRC(rc);
8573	return rc;
8574	}
8575
8576
8577	/**
8578	* Injects an event into the guest upon VM-entry by updating the relevant fields
8579	* in the VM-entry area in the VMCS.
8580	*
8581	* @returns Strict VBox status code (i.e. informational status codes too).
8582	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8583	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8584	*
8585	* @param pVCpu The cross context virtual CPU structure.
8586	* @param pVmxTransient The VMX-transient structure.
8587	* @param pEvent The event being injected.
8588	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8589	* will be updated if necessary. This cannot not be NULL.
8590	* @param fStepping Whether we're single-stepping guest execution and should
8591	* return VINF_EM_DBG_STEPPED if the event is injected
8592	* directly (registers modified by us, not by hardware on
8593	* VM-entry).
8594	*/
8595	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8596	uint32_t *pfIntrState)
8597	{
8598	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8599	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8600	Assert(pfIntrState);
8601
8602	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8603	uint32_t u32IntInfo = pEvent->u64IntInfo;
8604	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8605	uint32_t const cbInstr = pEvent->cbInstr;
8606	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8607	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8608	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8609
8610	#ifdef VBOX_STRICT
8611	/*
8612	* Validate the error-code-valid bit for hardware exceptions.
8613	* No error codes for exceptions in real-mode.
8614	*
8615	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8616	*/
8617	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8618	&& !CPUMIsGuestInRealModeEx(pCtx))
8619	{
8620	switch (uVector)
8621	{
8622	case X86_XCPT_PF:
8623	case X86_XCPT_DF:
8624	case X86_XCPT_TS:
8625	case X86_XCPT_NP:
8626	case X86_XCPT_SS:
8627	case X86_XCPT_GP:
8628	case X86_XCPT_AC:
8629	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8630	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8631	RT_FALL_THRU();
8632	default:
8633	break;
8634	}
8635	}
8636
8637	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8638	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8639	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8640	#endif
8641
8642	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8643	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
8644	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
8645	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
8646	{
8647	Assert(uVector <= X86_XCPT_LAST);
8648	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI \|\| uVector == X86_XCPT_NMI);
8649	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT \|\| uVector == X86_XCPT_DB);
8650	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedXcptsR0[uVector]);
8651	}
8652	else
8653	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8654
8655	/*
8656	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8657	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8658	* interrupt handler in the (real-mode) guest.
8659	*
8660	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8661	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8662	*/
8663	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8664	{
8665	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8666	{
8667	/*
8668	* For CPUs with unrestricted guest execution enabled and with the guest
8669	* in real-mode, we must not set the deliver-error-code bit.
8670	*
8671	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8672	*/
8673	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8674	}
8675	else
8676	{
8677	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8678	Assert(PDMVmmDevHeapIsEnabled(pVM));
8679	Assert(pVM->hm.s.vmx.pRealModeTSS);
8680	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8681
8682	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8683	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8684	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8685	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8686	AssertRCReturn(rc2, rc2);
8687
8688	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8689	size_t const cbIdtEntry = sizeof(X86IDTR16);
8690	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8691	{
8692	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8693	if (uVector == X86_XCPT_DF)
8694	return VINF_EM_RESET;
8695
8696	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8697	No error codes for exceptions in real-mode. */
8698	if (uVector == X86_XCPT_GP)
8699	{
8700	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8701	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8702	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8703	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8704	HMEVENT EventXcptDf;
8705	RT_ZERO(EventXcptDf);
8706	EventXcptDf.u64IntInfo = uXcptDfInfo;
8707	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8708	}
8709
8710	/*
8711	* If we're injecting an event with no valid IDT entry, inject a #GP.
8712	* No error codes for exceptions in real-mode.
8713	*
8714	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8715	*/
8716	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8717	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8718	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8719	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8720	HMEVENT EventXcptGp;
8721	RT_ZERO(EventXcptGp);
8722	EventXcptGp.u64IntInfo = uXcptGpInfo;
8723	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8724	}
8725
8726	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8727	uint16_t uGuestIp = pCtx->ip;
8728	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8729	{
8730	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8731	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8732	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8733	}
8734	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8735	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8736
8737	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8738	X86IDTR16 IdtEntry;
8739	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8740	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8741	AssertRCReturn(rc2, rc2);
8742
8743	/* Construct the stack frame for the interrupt/exception handler. */
8744	VBOXSTRICTRC rcStrict;
8745	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8746	if (rcStrict == VINF_SUCCESS)
8747	{
8748	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8749	if (rcStrict == VINF_SUCCESS)
8750	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8751	}
8752
8753	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8754	if (rcStrict == VINF_SUCCESS)
8755	{
8756	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8757	pCtx->rip = IdtEntry.offSel;
8758	pCtx->cs.Sel = IdtEntry.uSel;
8759	pCtx->cs.ValidSel = IdtEntry.uSel;
8760	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8761	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8762	&& uVector == X86_XCPT_PF)
8763	pCtx->cr2 = GCPtrFault;
8764
8765	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8766	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8767	\| HM_CHANGED_GUEST_RSP);
8768
8769	/*
8770	* If we delivered a hardware exception (other than an NMI) and if there was
8771	* block-by-STI in effect, we should clear it.
8772	*/
8773	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8774	{
8775	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8776	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8777	Log4Func(("Clearing inhibition due to STI\n"));
8778	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8779	}
8780
8781	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8782	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8783
8784	/*
8785	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8786	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8787	*/
8788	pVCpu->hm.s.Event.fPending = false;
8789
8790	/*
8791	* If we eventually support nested-guest execution without unrestricted guest execution,
8792	* we should set fInterceptEvents here.
8793	*/
8794	Assert(!pVmxTransient->fIsNestedGuest);
8795
8796	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8797	if (fStepping)
8798	rcStrict = VINF_EM_DBG_STEPPED;
8799	}
8800	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8801	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8802	return rcStrict;
8803	}
8804	}
8805
8806	/*
8807	* Validate.
8808	*/
8809	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8810	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8811
8812	/*
8813	* Inject the event into the VMCS.
8814	*/
8815	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8816	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8817	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8818	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8819	AssertRC(rc);
8820
8821	/*
8822	* Update guest CR2 if this is a page-fault.
8823	*/
8824	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8825	pCtx->cr2 = GCPtrFault;
8826
8827	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8828	return VINF_SUCCESS;
8829	}
8830
8831
8832	/**
8833	* Evaluates the event to be delivered to the guest and sets it as the pending
8834	* event.
8835	*
8836	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
8837	* exits to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must
8838	* NOT restore these force-flags.
8839	*
8840	* @returns Strict VBox status code (i.e. informational status codes too).
8841	* @param pVCpu The cross context virtual CPU structure.
8842	* @param pVmxTransient The VMX-transient structure.
8843	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8844	*/
8845	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8846	{
8847	Assert(pfIntrState);
8848	Assert(!TRPMHasTrap(pVCpu));
8849
8850	/*
8851	* Compute/update guest-interruptibility state related FFs.
8852	* The FFs will be used below while evaluating events to be injected.
8853	*/
8854	*pfIntrState = hmR0VmxGetGuestIntrStateAndUpdateFFs(pVCpu);
8855
8856	/*
8857	* Evaluate if a new event needs to be injected.
8858	* An event that's already pending has already performed all necessary checks.
8859	*/
8860	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8861	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8862	if ( !pVCpu->hm.s.Event.fPending
8863	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
8864	{
8865	/** @todo SMI. SMIs take priority over NMIs. */
8866
8867	/*
8868	* NMIs.
8869	* NMIs take priority over external interrupts.
8870	*/
8871	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8872	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8873	{
8874	/*
8875	* For a guest, the FF always indicates the guest's ability to receive an NMI.
8876	*
8877	* For a nested-guest, the FF always indicates the outer guest's ability to
8878	* receive an NMI while the guest-interruptibility state bit depends on whether
8879	* the nested-hypervisor is using virtual-NMIs.
8880	*/
8881	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
8882	{
8883	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8884	if ( fIsNestedGuest
8885	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
8886	return IEMExecVmxVmexitXcptNmi(pVCpu);
8887	#endif
8888	hmR0VmxSetPendingXcptNmi(pVCpu);
8889	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8890	Log4Func(("NMI pending injection\n"));
8891
8892	/* We've injected the NMI, bail. */
8893	return VINF_SUCCESS;
8894	}
8895	else if (!fIsNestedGuest)
8896	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8897	}
8898
8899	/*
8900	* External interrupts (PIC/APIC).
8901	* Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it.
8902	* We cannot re-request the interrupt from the controller again.
8903	*/
8904	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8905	&& !pVCpu->hm.s.fSingleInstruction)
8906	{
8907	Assert(!DBGFIsStepping(pVCpu));
8908	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8909	AssertRC(rc);
8910
8911	/*
8912	* We must not check EFLAGS directly when executing a nested-guest, use
8913	* CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of
8914	* external interrupts when "External interrupt exiting" is set. This fixes a nasty
8915	* SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by
8916	* other VM-exits (like a preemption timer), see @bugref{9562#c18}.
8917	*
8918	* See Intel spec. 25.4.1 "Event Blocking".
8919	*/
8920	if (CPUMIsGuestPhysIntrEnabled(pVCpu))
8921	{
8922	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8923	if ( fIsNestedGuest
8924	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8925	{
8926	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8927	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8928	return rcStrict;
8929	}
8930	#endif
8931	uint8_t u8Interrupt;
8932	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8933	if (RT_SUCCESS(rc))
8934	{
8935	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8936	if ( fIsNestedGuest
8937	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8938	{
8939	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8940	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8941	return rcStrict;
8942	}
8943	#endif
8944	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8945	Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
8946	}
8947	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8948	{
8949	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8950
8951	if ( !fIsNestedGuest
8952	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8953	hmR0VmxApicSetTprThreshold(pVmcsInfo, u8Interrupt >> 4);
8954	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8955
8956	/*
8957	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8958	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8959	* need to re-set this force-flag here.
8960	*/
8961	}
8962	else
8963	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8964
8965	/* We've injected the interrupt or taken necessary action, bail. */
8966	return VINF_SUCCESS;
8967	}
8968	else if (!fIsNestedGuest)
8969	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8970	}
8971	}
8972	else if (!fIsNestedGuest)
8973	{
8974	/*
8975	* An event is being injected or we are in an interrupt shadow. Check if another event is
8976	* pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept
8977	* the pending event.
8978	*/
8979	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8980	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8981	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8982	&& !pVCpu->hm.s.fSingleInstruction)
8983	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8984	}
8985	/* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */
8986
8987	return VINF_SUCCESS;
8988	}
8989
8990
8991	/**
8992	* Injects any pending events into the guest if the guest is in a state to
8993	* receive them.
8994	*
8995	* @returns Strict VBox status code (i.e. informational status codes too).
8996	* @param pVCpu The cross context virtual CPU structure.
8997	* @param pVmxTransient The VMX-transient structure.
8998	* @param fIntrState The VT-x guest-interruptibility state.
8999	* @param fStepping Whether we are single-stepping the guest using the
9000	* hypervisor debugger and should return
9001	* VINF_EM_DBG_STEPPED if the event was dispatched
9002	* directly.
9003	*/
9004	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9005	{
9006	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9007	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9008
9009	#ifdef VBOX_STRICT
9010	/*
9011	* Verify guest-interruptibility state.
9012	*
9013	* We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
9014	* since injecting an event may modify the interruptibility state and we must thus always
9015	* use fIntrState.
9016	*/
9017	{
9018	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9019	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9020	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9021	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9022	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9023	Assert(!TRPMHasTrap(pVCpu));
9024	NOREF(fBlockMovSS); NOREF(fBlockSti);
9025	}
9026	#endif
9027
9028	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9029	if (pVCpu->hm.s.Event.fPending)
9030	{
9031	/*
9032	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9033	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9034	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9035	*
9036	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9037	*/
9038	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9039	#ifdef VBOX_STRICT
9040	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9041	{
9042	Assert(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9043	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
9044	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9045	}
9046	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9047	{
9048	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
9049	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
9050	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9051	}
9052	#endif
9053	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9054	uIntType));
9055
9056	/*
9057	* Inject the event and get any changes to the guest-interruptibility state.
9058	*
9059	* The guest-interruptibility state may need to be updated if we inject the event
9060	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9061	*/
9062	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9063	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9064
9065	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9066	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9067	else
9068	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9069	}
9070
9071	/*
9072	* Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
9073	* is an interrupt shadow (block-by-STI or block-by-MOV SS).
9074	*/
9075	if ( (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9076	&& !pVmxTransient->fIsNestedGuest)
9077	{
9078	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9079
9080	if (!pVCpu->hm.s.fSingleInstruction)
9081	{
9082	/*
9083	* Set or clear the BS bit depending on whether the trap flag is active or not. We need
9084	* to do both since we clear the BS bit from the VMCS while exiting to ring-3.
9085	*/
9086	Assert(!DBGFIsStepping(pVCpu));
9087	uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_TF);
9088	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
9089	AssertRC(rc);
9090	}
9091	else
9092	{
9093	/*
9094	* We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
9095	* hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
9096	* we take care of this case in hmR0VmxExportSharedDebugState and also the case if
9097	* we use MTF, so just make sure it's called before executing guest-code.
9098	*/
9099	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
9100	}
9101	}
9102	/* else: for nested-guest currently handling while merging controls. */
9103
9104	/*
9105	* Finally, update the guest-interruptibility state.
9106	*
9107	* This is required for the real-on-v86 software interrupt injection, for
9108	* pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
9109	*/
9110	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9111	AssertRC(rc);
9112
9113	/*
9114	* There's no need to clear the VM-entry interruption-information field here if we're not
9115	* injecting anything. VT-x clears the valid bit on every VM-exit.
9116	*
9117	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9118	*/
9119
9120	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9121	return rcStrict;
9122	}
9123
9124
9125	/**
9126	* Enters the VT-x session.
9127	*
9128	* @returns VBox status code.
9129	* @param pVCpu The cross context virtual CPU structure.
9130	*/
9131	VMMR0DECL(int) VMXR0Enter(PVMCPUCC pVCpu)
9132	{
9133	AssertPtr(pVCpu);
9134	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9135	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9136
9137	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9138	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9139	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9140
9141	#ifdef VBOX_STRICT
9142	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9143	RTCCUINTREG uHostCr4 = ASMGetCR4();
9144	if (!(uHostCr4 & X86_CR4_VMXE))
9145	{
9146	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9147	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9148	}
9149	#endif
9150
9151	/*
9152	* Load the appropriate VMCS as the current and active one.
9153	*/
9154	PVMXVMCSINFO pVmcsInfo;
9155	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9156	if (!fInNestedGuestMode)
9157	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9158	else
9159	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9160	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9161	if (RT_SUCCESS(rc))
9162	{
9163	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9164	pVCpu->hm.s.fLeaveDone = false;
9165	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9166
9167	/*
9168	* Do the EMT scheduled L1D flush here if needed.
9169	*/
9170	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9171	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9172	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9173	hmR0MdsClear();
9174	}
9175	return rc;
9176	}
9177
9178
9179	/**
9180	* The thread-context callback (only on platforms which support it).
9181	*
9182	* @param enmEvent The thread-context event.
9183	* @param pVCpu The cross context virtual CPU structure.
9184	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9185	* @thread EMT(pVCpu)
9186	*/
9187	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
9188	{
9189	AssertPtr(pVCpu);
9190	RT_NOREF1(fGlobalInit);
9191
9192	switch (enmEvent)
9193	{
9194	case RTTHREADCTXEVENT_OUT:
9195	{
9196	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9197	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9198	VMCPU_ASSERT_EMT(pVCpu);
9199
9200	/* No longjmps (logger flushes, locks) in this fragile context. */
9201	VMMRZCallRing3Disable(pVCpu);
9202	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9203
9204	/* Restore host-state (FPU, debug etc.) */
9205	if (!pVCpu->hm.s.fLeaveDone)
9206	{
9207	/*
9208	* Do -not- import the guest-state here as we might already be in the middle of importing
9209	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9210	*/
9211	hmR0VmxLeave(pVCpu, false /* fImportState */);
9212	pVCpu->hm.s.fLeaveDone = true;
9213	}
9214
9215	/* Leave HM context, takes care of local init (term). */
9216	int rc = HMR0LeaveCpu(pVCpu);
9217	AssertRC(rc);
9218
9219	/* Restore longjmp state. */
9220	VMMRZCallRing3Enable(pVCpu);
9221	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9222	break;
9223	}
9224
9225	case RTTHREADCTXEVENT_IN:
9226	{
9227	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9228	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9229	VMCPU_ASSERT_EMT(pVCpu);
9230
9231	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9232	VMMRZCallRing3Disable(pVCpu);
9233	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9234
9235	/* Initialize the bare minimum state required for HM. This takes care of
9236	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9237	int rc = hmR0EnterCpu(pVCpu);
9238	AssertRC(rc);
9239	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9240	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9241
9242	/* Load the active VMCS as the current one. */
9243	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9244	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9245	AssertRC(rc);
9246	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9247	pVCpu->hm.s.fLeaveDone = false;
9248
9249	/* Do the EMT scheduled L1D flush if needed. */
9250	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9251	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9252
9253	/* Restore longjmp state. */
9254	VMMRZCallRing3Enable(pVCpu);
9255	break;
9256	}
9257
9258	default:
9259	break;
9260	}
9261	}
9262
9263
9264	/**
9265	* Exports the host state into the VMCS host-state area.
9266	* Sets up the VM-exit MSR-load area.
9267	*
9268	* The CPU state will be loaded from these fields on every successful VM-exit.
9269	*
9270	* @returns VBox status code.
9271	* @param pVCpu The cross context virtual CPU structure.
9272	*
9273	* @remarks No-long-jump zone!!!
9274	*/
9275	static int hmR0VmxExportHostState(PVMCPUCC pVCpu)
9276	{
9277	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9278
9279	int rc = VINF_SUCCESS;
9280	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9281	{
9282	hmR0VmxExportHostControlRegs();
9283
9284	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9285	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9286
9287	hmR0VmxExportHostMsrs(pVCpu);
9288
9289	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9290	}
9291	return rc;
9292	}
9293
9294
9295	/**
9296	* Saves the host state in the VMCS host-state.
9297	*
9298	* @returns VBox status code.
9299	* @param pVCpu The cross context virtual CPU structure.
9300	*
9301	* @remarks No-long-jump zone!!!
9302	*/
9303	VMMR0DECL(int) VMXR0ExportHostState(PVMCPUCC pVCpu)
9304	{
9305	AssertPtr(pVCpu);
9306	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9307
9308	/*
9309	* Export the host state here while entering HM context.
9310	* When thread-context hooks are used, we might get preempted and have to re-save the host
9311	* state but most of the time we won't be, so do it here before we disable interrupts.
9312	*/
9313	return hmR0VmxExportHostState(pVCpu);
9314	}
9315
9316
9317	/**
9318	* Exports the guest state into the VMCS guest-state area.
9319	*
9320	* The will typically be done before VM-entry when the guest-CPU state and the
9321	* VMCS state may potentially be out of sync.
9322	*
9323	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9324	* VM-entry controls.
9325	* Sets up the appropriate VMX non-root function to execute guest code based on
9326	* the guest CPU mode.
9327	*
9328	* @returns VBox strict status code.
9329	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9330	* without unrestricted guest execution and the VMMDev is not presently
9331	* mapped (e.g. EFI32).
9332	*
9333	* @param pVCpu The cross context virtual CPU structure.
9334	* @param pVmxTransient The VMX-transient structure.
9335	*
9336	* @remarks No-long-jump zone!!!
9337	*/
9338	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9339	{
9340	AssertPtr(pVCpu);
9341	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9342	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9343
9344	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9345
9346	/*
9347	* Determine real-on-v86 mode.
9348	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9349	*/
9350	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9351	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9352	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9353	pVmcsInfo->RealMode.fRealOnV86Active = false;
9354	else
9355	{
9356	Assert(!pVmxTransient->fIsNestedGuest);
9357	pVmcsInfo->RealMode.fRealOnV86Active = true;
9358	}
9359
9360	/*
9361	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9362	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9363	*/
9364	int rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9365	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9366
9367	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9368	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9369
9370	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9371	if (rcStrict == VINF_SUCCESS)
9372	{ /* likely */ }
9373	else
9374	{
9375	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9376	return rcStrict;
9377	}
9378
9379	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9380	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9381
9382	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9383	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9384
9385	hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9386	hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9387	hmR0VmxExportGuestRip(pVCpu);
9388	hmR0VmxExportGuestRsp(pVCpu);
9389	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9390
9391	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9392	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9393
9394	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9395	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9396	\| HM_CHANGED_GUEST_CR2
9397	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9398	\| HM_CHANGED_GUEST_X87
9399	\| HM_CHANGED_GUEST_SSE_AVX
9400	\| HM_CHANGED_GUEST_OTHER_XSAVE
9401	\| HM_CHANGED_GUEST_XCRx
9402	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9403	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9404	\| HM_CHANGED_GUEST_TSC_AUX
9405	\| HM_CHANGED_GUEST_OTHER_MSRS
9406	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9407
9408	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9409	return rc;
9410	}
9411
9412
9413	/**
9414	* Exports the state shared between the host and guest into the VMCS.
9415	*
9416	* @param pVCpu The cross context virtual CPU structure.
9417	* @param pVmxTransient The VMX-transient structure.
9418	*
9419	* @remarks No-long-jump zone!!!
9420	*/
9421	static void hmR0VmxExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9422	{
9423	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9424	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9425
9426	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9427	{
9428	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9429	AssertRC(rc);
9430	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9431
9432	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9433	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9434	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9435	}
9436
9437	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9438	{
9439	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9440	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9441	}
9442
9443	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9444	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9445	}
9446
9447
9448	/**
9449	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9450	*
9451	* @returns Strict VBox status code (i.e. informational status codes too).
9452	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9453	* without unrestricted guest execution and the VMMDev is not presently
9454	* mapped (e.g. EFI32).
9455	*
9456	* @param pVCpu The cross context virtual CPU structure.
9457	* @param pVmxTransient The VMX-transient structure.
9458	*
9459	* @remarks No-long-jump zone!!!
9460	*/
9461	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9462	{
9463	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9464	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9465	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9466
9467	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9468	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9469	#endif
9470
9471	/*
9472	* For many VM-exits only RIP/RSP/RFLAGS (and HWVIRT state when executing a nested-guest)
9473	* changes. First try to export only these without going through all other changed-flag checks.
9474	*/
9475	VBOXSTRICTRC rcStrict;
9476	uint64_t const fCtxMask = HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
9477	uint64_t const fMinimalMask = HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT;
9478	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9479
9480	/* If only RIP/RSP/RFLAGS/HWVIRT changed, export only those (quicker, happens more often).*/
9481	if ( (fCtxChanged & fMinimalMask)
9482	&& !(fCtxChanged & (fCtxMask & ~fMinimalMask)))
9483	{
9484	hmR0VmxExportGuestRip(pVCpu);
9485	hmR0VmxExportGuestRsp(pVCpu);
9486	hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9487	rcStrict = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9488	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9489	}
9490	/* If anything else also changed, go through the full export routine and export as required. */
9491	else if (fCtxChanged & fCtxMask)
9492	{
9493	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9494	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9495	{ /* likely */}
9496	else
9497	{
9498	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9499	VBOXSTRICTRC_VAL(rcStrict)));
9500	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9501	return rcStrict;
9502	}
9503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9504	}
9505	/* Nothing changed, nothing to load here. */
9506	else
9507	rcStrict = VINF_SUCCESS;
9508
9509	#ifdef VBOX_STRICT
9510	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9511	uint64_t const fCtxChangedCur = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9512	AssertMsg(!(fCtxChangedCur & fCtxMask), ("fCtxChangedCur=%#RX64\n", fCtxChangedCur));
9513	#endif
9514	return rcStrict;
9515	}
9516
9517
9518	/**
9519	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9520	* and update error record fields accordingly.
9521	*
9522	* @returns VMX_IGS_* error codes.
9523	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9524	* wrong with the guest state.
9525	*
9526	* @param pVCpu The cross context virtual CPU structure.
9527	* @param pVmcsInfo The VMCS info. object.
9528	*
9529	* @remarks This function assumes our cache of the VMCS controls
9530	* are valid, i.e. hmR0VmxCheckCachedVmcsCtls() succeeded.
9531	*/
9532	static uint32_t hmR0VmxCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9533	{
9534	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9535	#define HMVMX_CHECK_BREAK(expr, err) do { \
9536	if (!(expr)) { uError = (err); break; } \
9537	} while (0)
9538
9539	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9540	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9541	uint32_t uError = VMX_IGS_ERROR;
9542	uint32_t u32IntrState = 0;
9543	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9544	do
9545	{
9546	int rc;
9547
9548	/*
9549	* Guest-interruptibility state.
9550	*
9551	* Read this first so that any check that fails prior to those that actually
9552	* require the guest-interruptibility state would still reflect the correct
9553	* VMCS value and avoids causing further confusion.
9554	*/
9555	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9556	AssertRC(rc);
9557
9558	uint32_t u32Val;
9559	uint64_t u64Val;
9560
9561	/*
9562	* CR0.
9563	*/
9564	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9565	uint64_t fSetCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9566	uint64_t const fZapCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9567	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
9568	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9569	if (fUnrestrictedGuest)
9570	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
9571
9572	uint64_t u64GuestCr0;
9573	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64GuestCr0);
9574	AssertRC(rc);
9575	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9576	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9577	if ( !fUnrestrictedGuest
9578	&& (u64GuestCr0 & X86_CR0_PG)
9579	&& !(u64GuestCr0 & X86_CR0_PE))
9580	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9581
9582	/*
9583	* CR4.
9584	*/
9585	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9586	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9587	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9588
9589	uint64_t u64GuestCr4;
9590	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64GuestCr4);
9591	AssertRC(rc);
9592	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9593	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9594
9595	/*
9596	* IA32_DEBUGCTL MSR.
9597	*/
9598	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9599	AssertRC(rc);
9600	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9601	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9602	{
9603	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9604	}
9605	uint64_t u64DebugCtlMsr = u64Val;
9606
9607	#ifdef VBOX_STRICT
9608	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9609	AssertRC(rc);
9610	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9611	#endif
9612	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9613
9614	/*
9615	* RIP and RFLAGS.
9616	*/
9617	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
9618	AssertRC(rc);
9619	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9620	if ( !fLongModeGuest
9621	\|\| !pCtx->cs.Attr.n.u1Long)
9622	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9623	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9624	* must be identical if the "IA-32e mode guest" VM-entry
9625	* control is 1 and CS.L is 1. No check applies if the
9626	* CPU supports 64 linear-address bits. */
9627
9628	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9629	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9630	AssertRC(rc);
9631	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9632	VMX_IGS_RFLAGS_RESERVED);
9633	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9634	uint32_t const u32Eflags = u64Val;
9635
9636	if ( fLongModeGuest
9637	\|\| ( fUnrestrictedGuest
9638	&& !(u64GuestCr0 & X86_CR0_PE)))
9639	{
9640	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9641	}
9642
9643	uint32_t u32EntryInfo;
9644	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9645	AssertRC(rc);
9646	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9647	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9648
9649	/*
9650	* 64-bit checks.
9651	*/
9652	if (fLongModeGuest)
9653	{
9654	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9655	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9656	}
9657
9658	if ( !fLongModeGuest
9659	&& (u64GuestCr4 & X86_CR4_PCIDE))
9660	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9661
9662	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9663	* 51:32 beyond the processor's physical-address width are 0. */
9664
9665	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9666	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9667	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9668
9669	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9670	AssertRC(rc);
9671	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9672
9673	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9674	AssertRC(rc);
9675	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9676
9677	/*
9678	* PERF_GLOBAL MSR.
9679	*/
9680	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9681	{
9682	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9683	AssertRC(rc);
9684	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9685	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9686	}
9687
9688	/*
9689	* PAT MSR.
9690	*/
9691	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9692	{
9693	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9694	AssertRC(rc);
9695	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9696	for (unsigned i = 0; i < 8; i++)
9697	{
9698	uint8_t u8Val = (u64Val & 0xff);
9699	if ( u8Val != 0 /* UC */
9700	&& u8Val != 1 /* WC */
9701	&& u8Val != 4 /* WT */
9702	&& u8Val != 5 /* WP */
9703	&& u8Val != 6 /* WB */
9704	&& u8Val != 7 /* UC- */)
9705	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9706	u64Val >>= 8;
9707	}
9708	}
9709
9710	/*
9711	* EFER MSR.
9712	*/
9713	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9714	{
9715	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9716	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9717	AssertRC(rc);
9718	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9719	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9720	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9721	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9722	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9723	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9724	* iemVmxVmentryCheckGuestState(). */
9725	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9726	\|\| !(u64GuestCr0 & X86_CR0_PG)
9727	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9728	VMX_IGS_EFER_LMA_LME_MISMATCH);
9729	}
9730
9731	/*
9732	* Segment registers.
9733	*/
9734	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9735	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9736	if (!(u32Eflags & X86_EFL_VM))
9737	{
9738	/* CS */
9739	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9740	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9741	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9742	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9743	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9744	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9745	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9746	/* CS cannot be loaded with NULL in protected mode. */
9747	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9748	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9749	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9750	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9751	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9752	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9753	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9754	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9755	else
9756	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9757
9758	/* SS */
9759	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9760	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9761	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9762	if ( !(pCtx->cr0 & X86_CR0_PE)
9763	\|\| pCtx->cs.Attr.n.u4Type == 3)
9764	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9765
9766	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9767	{
9768	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9769	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9770	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9771	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9772	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9773	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9774	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9775	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9776	}
9777
9778	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9779	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9780	{
9781	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9782	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9783	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9784	\|\| pCtx->ds.Attr.n.u4Type > 11
9785	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9786	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9787	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9788	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9789	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9790	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9791	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9792	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9793	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9794	}
9795	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9796	{
9797	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9798	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9799	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9800	\|\| pCtx->es.Attr.n.u4Type > 11
9801	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9802	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9803	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9804	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9805	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9806	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9807	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9808	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9809	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9810	}
9811	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9812	{
9813	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9814	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9815	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9816	\|\| pCtx->fs.Attr.n.u4Type > 11
9817	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9818	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9819	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9820	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9821	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9822	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9823	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9824	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9825	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9826	}
9827	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9828	{
9829	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9830	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9831	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9832	\|\| pCtx->gs.Attr.n.u4Type > 11
9833	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9834	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9835	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9836	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9837	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9838	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9839	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9840	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9841	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9842	}
9843	/* 64-bit capable CPUs. */
9844	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9845	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9846	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9847	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9848	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9849	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9850	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9851	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9852	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9853	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9854	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9855	}
9856	else
9857	{
9858	/* V86 mode checks. */
9859	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9860	if (pVmcsInfo->RealMode.fRealOnV86Active)
9861	{
9862	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9863	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9864	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9865	}
9866	else
9867	{
9868	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9869	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9870	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9871	}
9872
9873	/* CS */
9874	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9875	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9876	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9877	/* SS */
9878	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9879	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9880	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9881	/* DS */
9882	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9883	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9884	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9885	/* ES */
9886	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9887	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9888	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9889	/* FS */
9890	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9891	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9892	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9893	/* GS */
9894	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9895	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9896	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9897	/* 64-bit capable CPUs. */
9898	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9899	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9900	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9901	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9902	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9903	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9904	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9905	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9906	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9907	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9908	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9909	}
9910
9911	/*
9912	* TR.
9913	*/
9914	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9915	/* 64-bit capable CPUs. */
9916	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9917	if (fLongModeGuest)
9918	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9919	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9920	else
9921	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9922	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9923	VMX_IGS_TR_ATTR_TYPE_INVALID);
9924	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9925	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9926	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9927	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9928	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9929	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9930	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9931	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9932
9933	/*
9934	* GDTR and IDTR (64-bit capable checks).
9935	*/
9936	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9937	AssertRC(rc);
9938	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9939
9940	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9941	AssertRC(rc);
9942	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9943
9944	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9945	AssertRC(rc);
9946	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9947
9948	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9949	AssertRC(rc);
9950	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9951
9952	/*
9953	* Guest Non-Register State.
9954	*/
9955	/* Activity State. */
9956	uint32_t u32ActivityState;
9957	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9958	AssertRC(rc);
9959	HMVMX_CHECK_BREAK( !u32ActivityState
9960	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9961	VMX_IGS_ACTIVITY_STATE_INVALID);
9962	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9963	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9964
9965	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9966	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9967	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9968
9969	/** @todo Activity state and injecting interrupts. Left as a todo since we
9970	* currently don't use activity states but ACTIVE. */
9971
9972	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9973	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9974
9975	/* Guest interruptibility-state. */
9976	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9977	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9978	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9979	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9980	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9981	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9982	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9983	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9984	{
9985	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9986	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9987	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9988	}
9989	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9990	{
9991	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9992	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9993	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9994	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9995	}
9996	/** @todo Assumes the processor is not in SMM. */
9997	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9998	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9999	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10000	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10001	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10002	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10003	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
10004	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10005
10006	/* Pending debug exceptions. */
10007	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10008	AssertRC(rc);
10009	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10010	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10011	u32Val = u64Val; /* For pending debug exceptions checks below. */
10012
10013	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10014	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10015	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10016	{
10017	if ( (u32Eflags & X86_EFL_TF)
10018	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10019	{
10020	/* Bit 14 is PendingDebug.BS. */
10021	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10022	}
10023	if ( !(u32Eflags & X86_EFL_TF)
10024	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10025	{
10026	/* Bit 14 is PendingDebug.BS. */
10027	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10028	}
10029	}
10030
10031	/* VMCS link pointer. */
10032	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10033	AssertRC(rc);
10034	if (u64Val != UINT64_C(0xffffffffffffffff))
10035	{
10036	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10037	/** @todo Bits beyond the processor's physical-address width MBZ. */
10038	/** @todo SMM checks. */
10039	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10040	Assert(pVmcsInfo->pvShadowVmcs);
10041	VMXVMCSREVID VmcsRevId;
10042	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10043	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10044	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10045	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10046	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10047	}
10048
10049	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10050	* not using nested paging? */
10051	if ( pVM->hm.s.fNestedPaging
10052	&& !fLongModeGuest
10053	&& CPUMIsGuestInPAEModeEx(pCtx))
10054	{
10055	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10056	AssertRC(rc);
10057	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10058
10059	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10060	AssertRC(rc);
10061	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10062
10063	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10064	AssertRC(rc);
10065	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10066
10067	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10068	AssertRC(rc);
10069	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10070	}
10071
10072	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10073	if (uError == VMX_IGS_ERROR)
10074	uError = VMX_IGS_REASON_NOT_FOUND;
10075	} while (0);
10076
10077	pVCpu->hm.s.u32HMError = uError;
10078	pVCpu->hm.s.vmx.LastError.u32GuestIntrState = u32IntrState;
10079	return uError;
10080
10081	#undef HMVMX_ERROR_BREAK
10082	#undef HMVMX_CHECK_BREAK
10083	}
10084
10085
10086	/**
10087	* Map the APIC-access page for virtualizing APIC accesses.
10088	*
10089	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10090	* this not done as part of exporting guest state, see @bugref{8721}.
10091	*
10092	* @returns VBox status code.
10093	* @param pVCpu The cross context virtual CPU structure.
10094	*/
10095	static int hmR0VmxMapHCApicAccessPage(PVMCPUCC pVCpu)
10096	{
10097	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10098	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10099
10100	Assert(PDMHasApic(pVM));
10101	Assert(u64MsrApicBase);
10102
10103	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10104	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10105
10106	/* Unalias the existing mapping. */
10107	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10108	AssertRCReturn(rc, rc);
10109
10110	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10111	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10112	rc = IOMR0MmioMapMmioHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10113	AssertRCReturn(rc, rc);
10114
10115	/* Update the per-VCPU cache of the APIC base MSR. */
10116	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10117	return VINF_SUCCESS;
10118	}
10119
10120
10121	/**
10122	* Worker function passed to RTMpOnSpecific() that is to be called on the target
10123	* CPU.
10124	*
10125	* @param idCpu The ID for the CPU the function is called on.
10126	* @param pvUser1 Null, not used.
10127	* @param pvUser2 Null, not used.
10128	*/
10129	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
10130	{
10131	RT_NOREF3(idCpu, pvUser1, pvUser2);
10132	VMXDispatchHostNmi();
10133	}
10134
10135
10136	/**
10137	* Dispatching an NMI on the host CPU that received it.
10138	*
10139	* @returns VBox status code.
10140	* @param pVCpu The cross context virtual CPU structure.
10141	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
10142	* executing when receiving the host NMI in VMX non-root
10143	* operation.
10144	*/
10145	static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
10146	{
10147	RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
10148	Assert(idCpu != NIL_RTCPUID);
10149
10150	/*
10151	* We don't want to delay dispatching the NMI any more than we have to. However,
10152	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
10153	* after executing guest or nested-guest code for the following reasons:
10154	*
10155	* - We would need to perform VMREADs with interrupts disabled and is orders of
10156	* magnitude worse when we run as a nested hypervisor without VMCS shadowing
10157	* supported by the host hypervisor.
10158	*
10159	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
10160	* longer period of time just for handling an edge case like host NMIs which do
10161	* not occur nearly as frequently as other VM-exits.
10162	*
10163	* Let's cover the most likely scenario first. Check if we are on the target CPU
10164	* and dispatch the NMI right away. This should be much faster than calling into
10165	* RTMpOnSpecific() machinery.
10166	*/
10167	bool fDispatched = false;
10168	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10169	if (idCpu == RTMpCpuId())
10170	{
10171	VMXDispatchHostNmi();
10172	fDispatched = true;
10173	}
10174	ASMSetFlags(fEFlags);
10175	if (fDispatched)
10176	{
10177	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10178	return VINF_SUCCESS;
10179	}
10180
10181	/*
10182	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
10183	* there should be no race or recursion even if we are unlucky enough to be preempted
10184	* (to the target CPU) without dispatching the host NMI above.
10185	*/
10186	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
10187	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
10188	}
10189
10190
10191	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10192	/**
10193	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10194	* nested-guest using hardware-assisted VMX.
10195	*
10196	* @param pVCpu The cross context virtual CPU structure.
10197	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10198	* @param pVmcsInfoGst The guest VMCS info. object.
10199	*/
10200	static void hmR0VmxMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10201	{
10202	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10203	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10204	Assert(pu64MsrBitmap);
10205
10206	/*
10207	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10208	* MSR that is intercepted by the guest is also intercepted while executing the
10209	* nested-guest using hardware-assisted VMX.
10210	*
10211	* Note! If the nested-guest is not using an MSR bitmap, every MSR must cause a
10212	* nested-guest VM-exit even if the outer guest is not intercepting some
10213	* MSRs. We cannot assume the caller has initialized the nested-guest
10214	* MSR bitmap in this case.
10215	*
10216	* The nested hypervisor may also switch whether it uses MSR bitmaps for
10217	* each of its VM-entry, hence initializing it once per-VM while setting
10218	* up the nested-guest VMCS is not sufficient.
10219	*/
10220	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10221	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10222	{
10223	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10224	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10225	Assert(pu64MsrBitmapNstGst);
10226	Assert(pu64MsrBitmapGst);
10227
10228	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10229	for (uint32_t i = 0; i < cFrags; i++)
10230	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10231	}
10232	else
10233	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10234	}
10235
10236
10237	/**
10238	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10239	* hardware-assisted VMX execution of the nested-guest.
10240	*
10241	* For a guest, we don't modify these controls once we set up the VMCS and hence
10242	* this function is never called.
10243	*
10244	* For nested-guests since the nested hypervisor provides these controls on every
10245	* nested-guest VM-entry and could potentially change them everytime we need to
10246	* merge them before every nested-guest VM-entry.
10247	*
10248	* @returns VBox status code.
10249	* @param pVCpu The cross context virtual CPU structure.
10250	*/
10251	static int hmR0VmxMergeVmcsNested(PVMCPUCC pVCpu)
10252	{
10253	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10254	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10255	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10256	Assert(pVmcsNstGst);
10257
10258	/*
10259	* Merge the controls with the requirements of the guest VMCS.
10260	*
10261	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10262	* VMCS with the features supported by the physical CPU as it's already done by the
10263	* VMLAUNCH/VMRESUME instruction emulation.
10264	*
10265	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10266	* derived from the VMX features supported by the physical CPU.
10267	*/
10268
10269	/* Pin-based VM-execution controls. */
10270	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10271
10272	/* Processor-based VM-execution controls. */
10273	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10274	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10275	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10276	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10277	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10278
10279	/* Secondary processor-based VM-execution controls. */
10280	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10281	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10282	\| VMX_PROC_CTLS2_INVPCID
10283	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10284	\| VMX_PROC_CTLS2_RDTSCP
10285	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10286	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10287	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10288	\| VMX_PROC_CTLS2_VMFUNC));
10289
10290	/*
10291	* VM-entry controls:
10292	* These controls contains state that depends on the nested-guest state (primarily
10293	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10294	* VM-exit. Although the nested hypervisor cannot change it, we need to in order to
10295	* properly continue executing the nested-guest if the EFER MSR changes but does not
10296	* cause a nested-guest VM-exits.
10297	*
10298	* VM-exit controls:
10299	* These controls specify the host state on return. We cannot use the controls from
10300	* the nested hypervisor state as is as it would contain the guest state rather than
10301	* the host state. Since the host state is subject to change (e.g. preemption, trips
10302	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10303	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10304	*
10305	* VM-entry MSR-load:
10306	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10307	* context by the VMLAUNCH/VMRESUME instruction emulation.
10308	*
10309	* VM-exit MSR-store:
10310	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10311	* back into the VM-exit MSR-store area.
10312	*
10313	* VM-exit MSR-load areas:
10314	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10315	* can entirely ignore what the nested hypervisor wants to load here.
10316	*/
10317
10318	/*
10319	* Exception bitmap.
10320	*
10321	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10322	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10323	* code more flexible if intercepting exceptions become more dynamic in the future we do
10324	* it as part of exporting the nested-guest state.
10325	*/
10326	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10327
10328	/*
10329	* CR0/CR4 guest/host mask.
10330	*
10331	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10332	* cause VM-exits, so we need to merge them here.
10333	*/
10334	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10335	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10336
10337	/*
10338	* Page-fault error-code mask and match.
10339	*
10340	* Although we require unrestricted guest execution (and thereby nested-paging) for
10341	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10342	* normally intercept #PFs, it might intercept them for debugging purposes.
10343	*
10344	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10345	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10346	*/
10347	uint32_t u32XcptPFMask;
10348	uint32_t u32XcptPFMatch;
10349	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10350	{
10351	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10352	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10353	}
10354	else
10355	{
10356	u32XcptPFMask = 0;
10357	u32XcptPFMatch = 0;
10358	}
10359
10360	/*
10361	* Pause-Loop exiting.
10362	*/
10363	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10364	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10365
10366	/*
10367	* Pending debug exceptions.
10368	* Currently just copy whatever the nested-guest provides us.
10369	*/
10370	uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
10371
10372	/*
10373	* I/O Bitmap.
10374	*
10375	* We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
10376	* intercept all I/O port accesses.
10377	*/
10378	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10379	Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
10380
10381	/*
10382	* VMCS shadowing.
10383	*
10384	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10385	* enabled while executing the nested-guest.
10386	*/
10387	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10388
10389	/*
10390	* APIC-access page.
10391	*/
10392	RTHCPHYS HCPhysApicAccess;
10393	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10394	{
10395	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10396	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10397
10398	/** @todo NSTVMX: This is not really correct but currently is required to make
10399	* things work. We need to re-enable the page handler when we fallback to
10400	* IEM execution of the nested-guest! */
10401	PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
10402
10403	void *pvPage;
10404	PGMPAGEMAPLOCK PgLockApicAccess;
10405	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
10406	if (RT_SUCCESS(rc))
10407	{
10408	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10409	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10410
10411	/** @todo Handle proper releasing of page-mapping lock later. */
10412	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
10413	}
10414	else
10415	return rc;
10416	}
10417	else
10418	HCPhysApicAccess = 0;
10419
10420	/*
10421	* Virtual-APIC page and TPR threshold.
10422	*/
10423	RTHCPHYS HCPhysVirtApic;
10424	uint32_t u32TprThreshold;
10425	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10426	{
10427	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10428	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10429
10430	void *pvPage;
10431	PGMPAGEMAPLOCK PgLockVirtApic;
10432	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
10433	if (RT_SUCCESS(rc))
10434	{
10435	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10436	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10437
10438	/** @todo Handle proper releasing of page-mapping lock later. */
10439	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
10440	}
10441	else
10442	return rc;
10443
10444	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10445	}
10446	else
10447	{
10448	HCPhysVirtApic = 0;
10449	u32TprThreshold = 0;
10450
10451	/*
10452	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10453	* used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
10454	* be taken care of by EPT/shadow paging.
10455	*/
10456	if (pVM->hm.s.fAllow64BitGuests)
10457	{
10458	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10459	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10460	}
10461	}
10462
10463	/*
10464	* Validate basic assumptions.
10465	*/
10466	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10467	Assert(pVM->hm.s.vmx.fUnrestrictedGuest);
10468	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10469	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10470
10471	/*
10472	* Commit it to the nested-guest VMCS.
10473	*/
10474	int rc = VINF_SUCCESS;
10475	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10476	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10477	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10478	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10479	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10480	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10481	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10482	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10483	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10484	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10485	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10486	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10487	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10488	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10489	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10490	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10491	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10492	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10493	{
10494	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10495	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10496	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10497	}
10498	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10499	{
10500	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10501	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10502	}
10503	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10504	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10505	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
10506	AssertRC(rc);
10507
10508	/*
10509	* Update the nested-guest VMCS cache.
10510	*/
10511	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10512	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10513	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10514	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10515	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10516	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10517	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10518	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10519	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10520
10521	/*
10522	* We need to flush the TLB if we are switching the APIC-access page address.
10523	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10524	*/
10525	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10526	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10527
10528	/*
10529	* MSR bitmap.
10530	*
10531	* The MSR bitmap address has already been initialized while setting up the nested-guest
10532	* VMCS, here we need to merge the MSR bitmaps.
10533	*/
10534	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10535	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10536
10537	return VINF_SUCCESS;
10538	}
10539	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10540
10541
10542	/**
10543	* Does the preparations before executing guest code in VT-x.
10544	*
10545	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10546	* recompiler/IEM. We must be cautious what we do here regarding committing
10547	* guest-state information into the VMCS assuming we assuredly execute the
10548	* guest in VT-x mode.
10549	*
10550	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10551	* the common-state (TRPM/forceflags), we must undo those changes so that the
10552	* recompiler/IEM can (and should) use them when it resumes guest execution.
10553	* Otherwise such operations must be done when we can no longer exit to ring-3.
10554	*
10555	* @returns Strict VBox status code (i.e. informational status codes too).
10556	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10557	* have been disabled.
10558	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10559	* pending events).
10560	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10561	* double-fault into the guest.
10562	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10563	* dispatched directly.
10564	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10565	*
10566	* @param pVCpu The cross context virtual CPU structure.
10567	* @param pVmxTransient The VMX-transient structure.
10568	* @param fStepping Whether we are single-stepping the guest in the
10569	* hypervisor debugger. Makes us ignore some of the reasons
10570	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10571	* if event dispatching took place.
10572	*/
10573	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10574	{
10575	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10576
10577	Log4Func(("fIsNested=%RTbool fStepping=%RTbool\n", pVmxTransient->fIsNestedGuest, fStepping));
10578
10579	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10580	if (pVmxTransient->fIsNestedGuest)
10581	{
10582	RT_NOREF2(pVCpu, fStepping);
10583	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10584	return VINF_EM_RESCHEDULE_REM;
10585	}
10586	#endif
10587
10588	/*
10589	* Check and process force flag actions, some of which might require us to go back to ring-3.
10590	*/
10591	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pVmxTransient, fStepping);
10592	if (rcStrict == VINF_SUCCESS)
10593	{
10594	/* FFs don't get set all the time. */
10595	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10596	if ( pVmxTransient->fIsNestedGuest
10597	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10598	{
10599	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10600	return VINF_VMX_VMEXIT;
10601	}
10602	#endif
10603	}
10604	else
10605	return rcStrict;
10606
10607	/*
10608	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10609	*/
10610	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10611	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10612	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10613	&& PDMHasApic(pVM))
10614	{
10615	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10616	AssertRCReturn(rc, rc);
10617	}
10618
10619	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10620	/*
10621	* Merge guest VMCS controls with the nested-guest VMCS controls.
10622	*
10623	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10624	* saved state), we should be okay with merging controls as we initialize the
10625	* guest VMCS controls as part of VM setup phase.
10626	*/
10627	if ( pVmxTransient->fIsNestedGuest
10628	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10629	{
10630	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10631	AssertRCReturn(rc, rc);
10632	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10633	}
10634	#endif
10635
10636	/*
10637	* Evaluate events to be injected into the guest.
10638	*
10639	* Events in TRPM can be injected without inspecting the guest state.
10640	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10641	* guest to cause a VM-exit the next time they are ready to receive the event.
10642	*
10643	* With nested-guests, evaluating pending events may cause VM-exits. Also, verify
10644	* that the event in TRPM that we will inject using hardware-assisted VMX is -not-
10645	* subject to interecption. Otherwise, we should have checked and injected them
10646	* manually elsewhere (IEM).
10647	*/
10648	if (TRPMHasTrap(pVCpu))
10649	{
10650	Assert(!pVmxTransient->fIsNestedGuest \|\| !CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
10651	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10652	}
10653
10654	uint32_t fIntrState;
10655	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10656
10657	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10658	/*
10659	* While evaluating pending events if something failed (unlikely) or if we were
10660	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10661	*/
10662	if (rcStrict != VINF_SUCCESS)
10663	return rcStrict;
10664	if ( pVmxTransient->fIsNestedGuest
10665	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10666	{
10667	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10668	return VINF_VMX_VMEXIT;
10669	}
10670	#else
10671	Assert(rcStrict == VINF_SUCCESS);
10672	#endif
10673
10674	/*
10675	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10676	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10677	* also result in triple-faulting the VM.
10678	*
10679	* With nested-guests, the above does not apply since unrestricted guest execution is a
10680	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10681	*/
10682	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10683	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10684	{ /* likely */ }
10685	else
10686	{
10687	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10688	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10689	return rcStrict;
10690	}
10691
10692	/*
10693	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10694	* import CR3 themselves. We will need to update them here, as even as late as the above
10695	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10696	* the below force flags to be set.
10697	*/
10698	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10699	{
10700	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10701	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10702	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10703	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10704	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10705	}
10706	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10707	{
10708	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10709	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10710	}
10711
10712	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10713	/* Paranoia. */
10714	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10715	#endif
10716
10717	/*
10718	* No longjmps to ring-3 from this point on!!!
10719	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10720	* This also disables flushing of the R0-logger instance (if any).
10721	*/
10722	VMMRZCallRing3Disable(pVCpu);
10723
10724	/*
10725	* Export the guest state bits.
10726	*
10727	* We cannot perform longjmps while loading the guest state because we do not preserve the
10728	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10729	* CPU migration.
10730	*
10731	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10732	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10733	*/
10734	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10735	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10736	{ /* likely */ }
10737	else
10738	{
10739	VMMRZCallRing3Enable(pVCpu);
10740	return rcStrict;
10741	}
10742
10743	/*
10744	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10745	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10746	* preemption disabled for a while. Since this is purely to aid the
10747	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10748	* disable interrupt on NT.
10749	*
10750	* We need to check for force-flags that could've possible been altered since we last
10751	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10752	* see @bugref{6398}).
10753	*
10754	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10755	* to ring-3 before executing guest code.
10756	*/
10757	pVmxTransient->fEFlags = ASMIntDisableFlags();
10758
10759	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10760	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10761	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10762	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10763	{
10764	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10765	{
10766	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10767	/*
10768	* If we are executing a nested-guest make sure that we should intercept subsequent
10769	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10770	* the VM-exit instruction emulation happy.
10771	*/
10772	if (pVmxTransient->fIsNestedGuest)
10773	CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
10774	#endif
10775
10776	/*
10777	* We've injected any pending events. This is really the point of no return (to ring-3).
10778	*
10779	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10780	* returns from this function, so do -not- enable them here.
10781	*/
10782	pVCpu->hm.s.Event.fPending = false;
10783	return VINF_SUCCESS;
10784	}
10785
10786	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10787	rcStrict = VINF_EM_RAW_INTERRUPT;
10788	}
10789	else
10790	{
10791	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10792	rcStrict = VINF_EM_RAW_TO_R3;
10793	}
10794
10795	ASMSetFlags(pVmxTransient->fEFlags);
10796	VMMRZCallRing3Enable(pVCpu);
10797
10798	return rcStrict;
10799	}
10800
10801
10802	/**
10803	* Final preparations before executing guest code using hardware-assisted VMX.
10804	*
10805	* We can no longer get preempted to a different host CPU and there are no returns
10806	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10807	* failures), this function is not intended to fail sans unrecoverable hardware
10808	* errors.
10809	*
10810	* @param pVCpu The cross context virtual CPU structure.
10811	* @param pVmxTransient The VMX-transient structure.
10812	*
10813	* @remarks Called with preemption disabled.
10814	* @remarks No-long-jump zone!!!
10815	*/
10816	static void hmR0VmxPreRunGuestCommitted(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10817	{
10818	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10819	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10820	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10821	Assert(!pVCpu->hm.s.Event.fPending);
10822
10823	/*
10824	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10825	*/
10826	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10827	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10828
10829	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10830	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10831	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10832	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10833
10834	if (!CPUMIsGuestFPUStateActive(pVCpu))
10835	{
10836	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10837	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10838	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10839	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10840	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10841	}
10842
10843	/*
10844	* Re-export the host state bits as we may've been preempted (only happens when
10845	* thread-context hooks are used or when the VM start function changes) or if
10846	* the host CR0 is modified while loading the guest FPU state above.
10847	*
10848	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10849	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10850	* see @bugref{8432}.
10851	*
10852	* This may also happen when switching to/from a nested-guest VMCS without leaving
10853	* ring-0.
10854	*/
10855	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10856	{
10857	hmR0VmxExportHostState(pVCpu);
10858	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10859	}
10860	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10861
10862	/*
10863	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10864	*/
10865	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10866	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10867	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10868
10869	/*
10870	* Store status of the shared guest/host debug state at the time of VM-entry.
10871	*/
10872	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10873	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10874
10875	/*
10876	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10877	* more than one conditional check. The post-run side of our code shall determine
10878	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10879	*/
10880	if (pVmcsInfo->pbVirtApic)
10881	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10882
10883	/*
10884	* Update the host MSRs values in the VM-exit MSR-load area.
10885	*/
10886	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10887	{
10888	if (pVmcsInfo->cExitMsrLoad > 0)
10889	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10890	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10891	}
10892
10893	/*
10894	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10895	* VMX-preemption timer based on the next virtual sync clock deadline.
10896	*/
10897	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10898	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10899	{
10900	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10901	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10902	}
10903
10904	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10905	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10906	if (!fIsRdtscIntercepted)
10907	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10908	else
10909	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10910
10911	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10912	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10913	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10914	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Record the error reporting info. with the current host CPU. */
10915	pVmcsInfo->idHostCpuState = idCurrentCpu; /* Record the CPU for which the host-state has been exported. */
10916	pVmcsInfo->idHostCpuExec = idCurrentCpu; /* Record the CPU on which we shall execute. */
10917
10918	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10919
10920	TMNotifyStartOfExecution(pVM, pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10921	as we're about to start executing the guest. */
10922
10923	/*
10924	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10925	*
10926	* This is done this late as updating the TSC offsetting/preemption timer above
10927	* figures out if we can skip intercepting RDTSCP by calculating the number of
10928	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10929	*/
10930	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10931	&& !fIsRdtscIntercepted)
10932	{
10933	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10934
10935	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10936	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10937	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10938	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10939	AssertRC(rc);
10940	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10941	pVmxTransient->fRemoveTscAuxMsr = true;
10942	}
10943
10944	#ifdef VBOX_STRICT
10945	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10946	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10947	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10948	AssertRC(hmR0VmxCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10949	#endif
10950
10951	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10952	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10953	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10954	* see @bugref{9180#c54}. */
10955	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10956	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10957	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10958	#endif
10959	}
10960
10961
10962	/**
10963	* First C routine invoked after running guest code using hardware-assisted VMX.
10964	*
10965	* @param pVCpu The cross context virtual CPU structure.
10966	* @param pVmxTransient The VMX-transient structure.
10967	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10968	*
10969	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10970	*
10971	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10972	* unconditionally when it is safe to do so.
10973	*/
10974	static void hmR0VmxPostRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10975	{
10976	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10977
10978	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10979	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10980	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10981	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10982	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10983	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10984
10985	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10986	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10987	{
10988	uint64_t uGstTsc;
10989	if (!pVmxTransient->fIsNestedGuest)
10990	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10991	else
10992	{
10993	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10994	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10995	}
10996	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10997	}
10998
10999	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11000	TMNotifyEndOfExecution(pVCpu->CTX_SUFF(pVM), pVCpu); /* Notify TM that the guest is no longer running. */
11001	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11002
11003	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11004	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11005	#ifdef VBOX_STRICT
11006	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11007	#endif
11008	Assert(!ASMIntAreEnabled());
11009	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11010	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11011
11012	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
11013	/*
11014	* Clean all the VMCS fields in the transient structure before reading
11015	* anything from the VMCS.
11016	*/
11017	pVmxTransient->uExitReason = 0;
11018	pVmxTransient->uExitIntErrorCode = 0;
11019	pVmxTransient->uExitQual = 0;
11020	pVmxTransient->uGuestLinearAddr = 0;
11021	pVmxTransient->uExitIntInfo = 0;
11022	pVmxTransient->cbExitInstr = 0;
11023	pVmxTransient->ExitInstrInfo.u = 0;
11024	pVmxTransient->uEntryIntInfo = 0;
11025	pVmxTransient->uEntryXcptErrorCode = 0;
11026	pVmxTransient->cbEntryInstr = 0;
11027	pVmxTransient->uIdtVectoringInfo = 0;
11028	pVmxTransient->uIdtVectoringErrorCode = 0;
11029	#endif
11030
11031	/*
11032	* Save the basic VM-exit reason and check if the VM-entry failed.
11033	* See Intel spec. 24.9.1 "Basic VM-exit Information".
11034	*/
11035	uint32_t uExitReason;
11036	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11037	AssertRC(rc);
11038	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11039	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11040
11041	/*
11042	* Log the VM-exit before logging anything else as otherwise it might be a
11043	* tad confusing what happens before and after the world-switch.
11044	*/
11045	HMVMX_LOG_EXIT(pVCpu, uExitReason);
11046
11047	/*
11048	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11049	* bitmap permissions, if it was added before VM-entry.
11050	*/
11051	if (pVmxTransient->fRemoveTscAuxMsr)
11052	{
11053	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11054	pVmxTransient->fRemoveTscAuxMsr = false;
11055	}
11056
11057	/*
11058	* Check if VMLAUNCH/VMRESUME succeeded.
11059	* If this failed, we cause a guru meditation and cease further execution.
11060	*
11061	* However, if we are executing a nested-guest we might fail if we use the
11062	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11063	*/
11064	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11065	{
11066	/*
11067	* Update the VM-exit history array here even if the VM-entry failed due to:
11068	* - Invalid guest state.
11069	* - MSR loading.
11070	* - Machine-check event.
11071	*
11072	* In any of the above cases we will still have a "valid" VM-exit reason
11073	* despite @a fVMEntryFailed being false.
11074	*
11075	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11076	*
11077	* Note! We don't have CS or RIP at this point. Will probably address that later
11078	* by amending the history entry added here.
11079	*/
11080	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11081	UINT64_MAX, uHostTsc);
11082
11083	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11084	{
11085	VMMRZCallRing3Enable(pVCpu);
11086
11087	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11088	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11089
11090	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
11091	hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
11092	#endif
11093
11094	/*
11095	* Import the guest-interruptibility state always as we need it while evaluating
11096	* injecting events on re-entry.
11097	*
11098	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11099	* checking for real-mode while exporting the state because all bits that cause
11100	* mode changes wrt CR0 are intercepted.
11101	*/
11102	uint64_t const fImportMask = CPUMCTX_EXTRN_HM_VMX_INT_STATE
11103	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11104	\| HMVMX_CPUMCTX_EXTRN_ALL
11105	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11106	\| CPUMCTX_EXTRN_RFLAGS
11107	#endif
11108	;
11109	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImportMask);
11110	AssertRC(rc);
11111
11112	/*
11113	* Sync the TPR shadow with our APIC state.
11114	*/
11115	if ( !pVmxTransient->fIsNestedGuest
11116	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
11117	{
11118	Assert(pVmcsInfo->pbVirtApic);
11119	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11120	{
11121	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11122	AssertRC(rc);
11123	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11124	}
11125	}
11126
11127	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11128	return;
11129	}
11130	}
11131	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11132	else if (pVmxTransient->fIsNestedGuest)
11133	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11134	#endif
11135	else
11136	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11137
11138	VMMRZCallRing3Enable(pVCpu);
11139	}
11140
11141
11142	/**
11143	* Runs the guest code using hardware-assisted VMX the normal way.
11144	*
11145	* @returns VBox status code.
11146	* @param pVCpu The cross context virtual CPU structure.
11147	* @param pcLoops Pointer to the number of executed loops.
11148	*/
11149	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
11150	{
11151	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11152	Assert(pcLoops);
11153	Assert(*pcLoops <= cMaxResumeLoops);
11154	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11155
11156	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11157	/*
11158	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11159	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11160	* guest VMCS while entering the VMX ring-0 session.
11161	*/
11162	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11163	{
11164	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11165	if (RT_SUCCESS(rc))
11166	{ /* likely */ }
11167	else
11168	{
11169	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11170	return rc;
11171	}
11172	}
11173	#endif
11174
11175	VMXTRANSIENT VmxTransient;
11176	RT_ZERO(VmxTransient);
11177	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11178
11179	/* Paranoia. */
11180	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11181
11182	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11183	for (;;)
11184	{
11185	Assert(!HMR0SuspendPending());
11186	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11187	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11188
11189	/*
11190	* Preparatory work for running nested-guest code, this may force us to
11191	* return to ring-3.
11192	*
11193	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11194	*/
11195	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11196	if (rcStrict != VINF_SUCCESS)
11197	break;
11198
11199	/* Interrupts are disabled at this point! */
11200	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11201	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11202	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11203	/* Interrupts are re-enabled at this point! */
11204
11205	/*
11206	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11207	*/
11208	if (RT_SUCCESS(rcRun))
11209	{ /* very likely */ }
11210	else
11211	{
11212	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11213	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11214	return rcRun;
11215	}
11216
11217	/*
11218	* Profile the VM-exit.
11219	*/
11220	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11221	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11222	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11223	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11224	HMVMX_START_EXIT_DISPATCH_PROF();
11225
11226	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11227
11228	/*
11229	* Handle the VM-exit.
11230	*/
11231	#ifdef HMVMX_USE_FUNCTION_TABLE
11232	rcStrict = g_aVMExitHandlers[VmxTransient.uExitReason].pfn(pVCpu, &VmxTransient);
11233	#else
11234	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11235	#endif
11236	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11237	if (rcStrict == VINF_SUCCESS)
11238	{
11239	if (++(*pcLoops) <= cMaxResumeLoops)
11240	continue;
11241	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11242	rcStrict = VINF_EM_RAW_INTERRUPT;
11243	}
11244	break;
11245	}
11246
11247	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11248	return rcStrict;
11249	}
11250
11251
11252	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11253	/**
11254	* Runs the nested-guest code using hardware-assisted VMX.
11255	*
11256	* @returns VBox status code.
11257	* @param pVCpu The cross context virtual CPU structure.
11258	* @param pcLoops Pointer to the number of executed loops.
11259	*
11260	* @sa hmR0VmxRunGuestCodeNormal.
11261	*/
11262	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
11263	{
11264	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11265	Assert(pcLoops);
11266	Assert(*pcLoops <= cMaxResumeLoops);
11267	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11268
11269	/*
11270	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11271	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11272	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11273	*/
11274	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11275	{
11276	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11277	if (RT_SUCCESS(rc))
11278	{ /* likely */ }
11279	else
11280	{
11281	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11282	return rc;
11283	}
11284	}
11285
11286	VMXTRANSIENT VmxTransient;
11287	RT_ZERO(VmxTransient);
11288	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11289	VmxTransient.fIsNestedGuest = true;
11290
11291	/* Paranoia. */
11292	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11293
11294	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11295	for (;;)
11296	{
11297	Assert(!HMR0SuspendPending());
11298	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11299	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11300
11301	/*
11302	* Preparatory work for running guest code, this may force us to
11303	* return to ring-3.
11304	*
11305	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11306	*/
11307	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11308	if (rcStrict != VINF_SUCCESS)
11309	break;
11310
11311	/* Interrupts are disabled at this point! */
11312	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11313	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11314	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11315	/* Interrupts are re-enabled at this point! */
11316
11317	/*
11318	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11319	*/
11320	if (RT_SUCCESS(rcRun))
11321	{ /* very likely */ }
11322	else
11323	{
11324	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11325	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11326	return rcRun;
11327	}
11328
11329	/*
11330	* Profile the VM-exit.
11331	*/
11332	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11334	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11335	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11336	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11337	HMVMX_START_EXIT_DISPATCH_PROF();
11338
11339	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11340
11341	/*
11342	* Handle the VM-exit.
11343	*/
11344	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11345	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11346	if (rcStrict == VINF_SUCCESS)
11347	{
11348	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11349	{
11350	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11351	rcStrict = VINF_VMX_VMEXIT;
11352	}
11353	else
11354	{
11355	if (++(*pcLoops) <= cMaxResumeLoops)
11356	continue;
11357	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11358	rcStrict = VINF_EM_RAW_INTERRUPT;
11359	}
11360	}
11361	else
11362	Assert(rcStrict != VINF_VMX_VMEXIT);
11363	break;
11364	}
11365
11366	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11367	return rcStrict;
11368	}
11369	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11370
11371
11372	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11373	* probes.
11374	*
11375	* The following few functions and associated structure contains the bloat
11376	* necessary for providing detailed debug events and dtrace probes as well as
11377	* reliable host side single stepping. This works on the principle of
11378	* "subclassing" the normal execution loop and workers. We replace the loop
11379	* method completely and override selected helpers to add necessary adjustments
11380	* to their core operation.
11381	*
11382	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11383	* any performance for debug and analysis features.
11384	*
11385	* @{
11386	*/
11387
11388	/**
11389	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11390	* the debug run loop.
11391	*/
11392	typedef struct VMXRUNDBGSTATE
11393	{
11394	/** The RIP we started executing at. This is for detecting that we stepped. */
11395	uint64_t uRipStart;
11396	/** The CS we started executing with. */
11397	uint16_t uCsStart;
11398
11399	/** Whether we've actually modified the 1st execution control field. */
11400	bool fModifiedProcCtls : 1;
11401	/** Whether we've actually modified the 2nd execution control field. */
11402	bool fModifiedProcCtls2 : 1;
11403	/** Whether we've actually modified the exception bitmap. */
11404	bool fModifiedXcptBitmap : 1;
11405
11406	/** We desire the modified the CR0 mask to be cleared. */
11407	bool fClearCr0Mask : 1;
11408	/** We desire the modified the CR4 mask to be cleared. */
11409	bool fClearCr4Mask : 1;
11410	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11411	uint32_t fCpe1Extra;
11412	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11413	uint32_t fCpe1Unwanted;
11414	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11415	uint32_t fCpe2Extra;
11416	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11417	uint32_t bmXcptExtra;
11418	/** The sequence number of the Dtrace provider settings the state was
11419	* configured against. */
11420	uint32_t uDtraceSettingsSeqNo;
11421	/** VM-exits to check (one bit per VM-exit). */
11422	uint32_t bmExitsToCheck[3];
11423
11424	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11425	uint32_t fProcCtlsInitial;
11426	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11427	uint32_t fProcCtls2Initial;
11428	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11429	uint32_t bmXcptInitial;
11430	} VMXRUNDBGSTATE;
11431	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11432	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11433
11434
11435	/**
11436	* Initializes the VMXRUNDBGSTATE structure.
11437	*
11438	* @param pVCpu The cross context virtual CPU structure of the
11439	* calling EMT.
11440	* @param pVmxTransient The VMX-transient structure.
11441	* @param pDbgState The debug state to initialize.
11442	*/
11443	static void hmR0VmxRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11444	{
11445	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11446	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11447
11448	pDbgState->fModifiedProcCtls = false;
11449	pDbgState->fModifiedProcCtls2 = false;
11450	pDbgState->fModifiedXcptBitmap = false;
11451	pDbgState->fClearCr0Mask = false;
11452	pDbgState->fClearCr4Mask = false;
11453	pDbgState->fCpe1Extra = 0;
11454	pDbgState->fCpe1Unwanted = 0;
11455	pDbgState->fCpe2Extra = 0;
11456	pDbgState->bmXcptExtra = 0;
11457	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11458	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11459	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11460	}
11461
11462
11463	/**
11464	* Updates the VMSC fields with changes requested by @a pDbgState.
11465	*
11466	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11467	* immediately before executing guest code, i.e. when interrupts are disabled.
11468	* We don't check status codes here as we cannot easily assert or return in the
11469	* latter case.
11470	*
11471	* @param pVCpu The cross context virtual CPU structure.
11472	* @param pVmxTransient The VMX-transient structure.
11473	* @param pDbgState The debug state.
11474	*/
11475	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11476	{
11477	/*
11478	* Ensure desired flags in VMCS control fields are set.
11479	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11480	*
11481	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11482	* there should be no stale data in pCtx at this point.
11483	*/
11484	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11485	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11486	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11487	{
11488	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11489	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11490	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11491	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11492	pDbgState->fModifiedProcCtls = true;
11493	}
11494
11495	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11496	{
11497	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11498	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11499	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11500	pDbgState->fModifiedProcCtls2 = true;
11501	}
11502
11503	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11504	{
11505	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11506	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11507	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11508	pDbgState->fModifiedXcptBitmap = true;
11509	}
11510
11511	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11512	{
11513	pVmcsInfo->u64Cr0Mask = 0;
11514	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, 0);
11515	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11516	}
11517
11518	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11519	{
11520	pVmcsInfo->u64Cr4Mask = 0;
11521	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, 0);
11522	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11523	}
11524
11525	NOREF(pVCpu);
11526	}
11527
11528
11529	/**
11530	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11531	* re-entry next time around.
11532	*
11533	* @returns Strict VBox status code (i.e. informational status codes too).
11534	* @param pVCpu The cross context virtual CPU structure.
11535	* @param pVmxTransient The VMX-transient structure.
11536	* @param pDbgState The debug state.
11537	* @param rcStrict The return code from executing the guest using single
11538	* stepping.
11539	*/
11540	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11541	VBOXSTRICTRC rcStrict)
11542	{
11543	/*
11544	* Restore VM-exit control settings as we may not reenter this function the
11545	* next time around.
11546	*/
11547	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11548
11549	/* We reload the initial value, trigger what we can of recalculations the
11550	next time around. From the looks of things, that's all that's required atm. */
11551	if (pDbgState->fModifiedProcCtls)
11552	{
11553	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11554	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11555	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11556	AssertRC(rc2);
11557	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11558	}
11559
11560	/* We're currently the only ones messing with this one, so just restore the
11561	cached value and reload the field. */
11562	if ( pDbgState->fModifiedProcCtls2
11563	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11564	{
11565	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11566	AssertRC(rc2);
11567	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11568	}
11569
11570	/* If we've modified the exception bitmap, we restore it and trigger
11571	reloading and partial recalculation the next time around. */
11572	if (pDbgState->fModifiedXcptBitmap)
11573	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11574
11575	return rcStrict;
11576	}
11577
11578
11579	/**
11580	* Configures VM-exit controls for current DBGF and DTrace settings.
11581	*
11582	* This updates @a pDbgState and the VMCS execution control fields to reflect
11583	* the necessary VM-exits demanded by DBGF and DTrace.
11584	*
11585	* @param pVCpu The cross context virtual CPU structure.
11586	* @param pVmxTransient The VMX-transient structure. May update
11587	* fUpdatedTscOffsettingAndPreemptTimer.
11588	* @param pDbgState The debug state.
11589	*/
11590	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11591	{
11592	/*
11593	* Take down the dtrace serial number so we can spot changes.
11594	*/
11595	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11596	ASMCompilerBarrier();
11597
11598	/*
11599	* We'll rebuild most of the middle block of data members (holding the
11600	* current settings) as we go along here, so start by clearing it all.
11601	*/
11602	pDbgState->bmXcptExtra = 0;
11603	pDbgState->fCpe1Extra = 0;
11604	pDbgState->fCpe1Unwanted = 0;
11605	pDbgState->fCpe2Extra = 0;
11606	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11607	pDbgState->bmExitsToCheck[i] = 0;
11608
11609	/*
11610	* Software interrupts (INT XXh) - no idea how to trigger these...
11611	*/
11612	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11613	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11614	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11615	{
11616	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11617	}
11618
11619	/*
11620	* INT3 breakpoints - triggered by #BP exceptions.
11621	*/
11622	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11623	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11624
11625	/*
11626	* Exception bitmap and XCPT events+probes.
11627	*/
11628	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11629	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11630	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11631
11632	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11633	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11634	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11635	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11636	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11637	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11638	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11639	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11640	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11641	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11642	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11643	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11644	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11645	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11646	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11647	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11648	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11649	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11650
11651	if (pDbgState->bmXcptExtra)
11652	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11653
11654	/*
11655	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11656	*
11657	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11658	* So, when adding/changing/removing please don't forget to update it.
11659	*
11660	* Some of the macros are picking up local variables to save horizontal space,
11661	* (being able to see it in a table is the lesser evil here).
11662	*/
11663	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11664	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11665	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11666	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11667	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11668	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11669	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11670	} else do { } while (0)
11671	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11672	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11673	{ \
11674	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11675	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11676	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11677	} else do { } while (0)
11678	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11679	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11680	{ \
11681	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11682	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11683	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11684	} else do { } while (0)
11685	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11686	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11687	{ \
11688	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11689	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11690	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11691	} else do { } while (0)
11692
11693	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11694	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11695	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11696	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11697	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11698
11699	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11700	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11701	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11702	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11703	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11704	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11705	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11706	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11707	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11708	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11709	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11710	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11711	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11712	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11713	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11714	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11715	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11716	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11717	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11718	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11719	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11720	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11721	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11722	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11723	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11724	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11725	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11726	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11727	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11728	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11729	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11730	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11731	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11732	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11733	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11734	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11735
11736	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11737	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11738	{
11739	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11740	\| CPUMCTX_EXTRN_APIC_TPR);
11741	AssertRC(rc);
11742
11743	#if 0 /** @todo fix me */
11744	pDbgState->fClearCr0Mask = true;
11745	pDbgState->fClearCr4Mask = true;
11746	#endif
11747	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11748	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11749	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11750	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11751	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11752	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11753	require clearing here and in the loop if we start using it. */
11754	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11755	}
11756	else
11757	{
11758	if (pDbgState->fClearCr0Mask)
11759	{
11760	pDbgState->fClearCr0Mask = false;
11761	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11762	}
11763	if (pDbgState->fClearCr4Mask)
11764	{
11765	pDbgState->fClearCr4Mask = false;
11766	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11767	}
11768	}
11769	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11770	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11771
11772	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11773	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11774	{
11775	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11776	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11777	}
11778	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11779	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11780
11781	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11782	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11783	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11784	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11785	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11786	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11787	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11788	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11789	#if 0 /** @todo too slow, fix handler. */
11790	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11791	#endif
11792	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11793
11794	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11795	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11796	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11797	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11798	{
11799	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11800	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11801	}
11802	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11803	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11804	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11805	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11806
11807	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11808	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11809	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11810	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11811	{
11812	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11813	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11814	}
11815	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11816	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11817	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11818	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11819
11820	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11821	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11822	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11823	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11824	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11825	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11826	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11827	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11828	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11829	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11830	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11831	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11832	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11833	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11834	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11835	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11836	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11837	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11838	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11839	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11840	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11841	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11842
11843	#undef IS_EITHER_ENABLED
11844	#undef SET_ONLY_XBM_IF_EITHER_EN
11845	#undef SET_CPE1_XBM_IF_EITHER_EN
11846	#undef SET_CPEU_XBM_IF_EITHER_EN
11847	#undef SET_CPE2_XBM_IF_EITHER_EN
11848
11849	/*
11850	* Sanitize the control stuff.
11851	*/
11852	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11853	if (pDbgState->fCpe2Extra)
11854	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11855	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11856	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11857	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11858	{
11859	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11860	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11861	}
11862
11863	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11864	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11865	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11866	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11867	}
11868
11869
11870	/**
11871	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11872	* appropriate.
11873	*
11874	* The caller has checked the VM-exit against the
11875	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11876	* already, so we don't have to do that either.
11877	*
11878	* @returns Strict VBox status code (i.e. informational status codes too).
11879	* @param pVCpu The cross context virtual CPU structure.
11880	* @param pVmxTransient The VMX-transient structure.
11881	* @param uExitReason The VM-exit reason.
11882	*
11883	* @remarks The name of this function is displayed by dtrace, so keep it short
11884	* and to the point. No longer than 33 chars long, please.
11885	*/
11886	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11887	{
11888	/*
11889	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11890	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11891	*
11892	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11893	* does. Must add/change/remove both places. Same ordering, please.
11894	*
11895	* Added/removed events must also be reflected in the next section
11896	* where we dispatch dtrace events.
11897	*/
11898	bool fDtrace1 = false;
11899	bool fDtrace2 = false;
11900	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11901	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11902	uint32_t uEventArg = 0;
11903	#define SET_EXIT(a_EventSubName) \
11904	do { \
11905	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11906	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11907	} while (0)
11908	#define SET_BOTH(a_EventSubName) \
11909	do { \
11910	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11911	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11912	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11913	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11914	} while (0)
11915	switch (uExitReason)
11916	{
11917	case VMX_EXIT_MTF:
11918	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11919
11920	case VMX_EXIT_XCPT_OR_NMI:
11921	{
11922	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11923	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11924	{
11925	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11926	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11927	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11928	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11929	{
11930	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11931	{
11932	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11933	uEventArg = pVmxTransient->uExitIntErrorCode;
11934	}
11935	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11936	switch (enmEvent1)
11937	{
11938	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11939	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11940	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11941	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11942	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11943	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11944	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11945	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11946	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11947	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11948	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11949	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11950	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11951	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11952	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11953	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11954	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11955	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11956	default: break;
11957	}
11958	}
11959	else
11960	AssertFailed();
11961	break;
11962
11963	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11964	uEventArg = idxVector;
11965	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11966	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11967	break;
11968	}
11969	break;
11970	}
11971
11972	case VMX_EXIT_TRIPLE_FAULT:
11973	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11974	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11975	break;
11976	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11977	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11978	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11979	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11980	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11981
11982	/* Instruction specific VM-exits: */
11983	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11984	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11985	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11986	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11987	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11988	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11989	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11990	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11991	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11992	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11993	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11994	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11995	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11996	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11997	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11998	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11999	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12000	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12001	case VMX_EXIT_MOV_CRX:
12002	hmR0VmxReadExitQualVmcs(pVmxTransient);
12003	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12004	SET_BOTH(CRX_READ);
12005	else
12006	SET_BOTH(CRX_WRITE);
12007	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12008	break;
12009	case VMX_EXIT_MOV_DRX:
12010	hmR0VmxReadExitQualVmcs(pVmxTransient);
12011	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12012	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12013	SET_BOTH(DRX_READ);
12014	else
12015	SET_BOTH(DRX_WRITE);
12016	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12017	break;
12018	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12019	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12020	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12021	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12022	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12023	case VMX_EXIT_GDTR_IDTR_ACCESS:
12024	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12025	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12026	{
12027	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12028	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12029	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12030	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12031	}
12032	break;
12033
12034	case VMX_EXIT_LDTR_TR_ACCESS:
12035	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12036	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12037	{
12038	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12039	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12040	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12041	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12042	}
12043	break;
12044
12045	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12046	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12047	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12048	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12049	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12050	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12051	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12052	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12053	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12054	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12055	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12056
12057	/* Events that aren't relevant at this point. */
12058	case VMX_EXIT_EXT_INT:
12059	case VMX_EXIT_INT_WINDOW:
12060	case VMX_EXIT_NMI_WINDOW:
12061	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12062	case VMX_EXIT_PREEMPT_TIMER:
12063	case VMX_EXIT_IO_INSTR:
12064	break;
12065
12066	/* Errors and unexpected events. */
12067	case VMX_EXIT_INIT_SIGNAL:
12068	case VMX_EXIT_SIPI:
12069	case VMX_EXIT_IO_SMI:
12070	case VMX_EXIT_SMI:
12071	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12072	case VMX_EXIT_ERR_MSR_LOAD:
12073	case VMX_EXIT_ERR_MACHINE_CHECK:
12074	case VMX_EXIT_PML_FULL:
12075	case VMX_EXIT_VIRTUALIZED_EOI:
12076	break;
12077
12078	default:
12079	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12080	break;
12081	}
12082	#undef SET_BOTH
12083	#undef SET_EXIT
12084
12085	/*
12086	* Dtrace tracepoints go first. We do them here at once so we don't
12087	* have to copy the guest state saving and stuff a few dozen times.
12088	* Down side is that we've got to repeat the switch, though this time
12089	* we use enmEvent since the probes are a subset of what DBGF does.
12090	*/
12091	if (fDtrace1 \|\| fDtrace2)
12092	{
12093	hmR0VmxReadExitQualVmcs(pVmxTransient);
12094	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12095	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12096	switch (enmEvent1)
12097	{
12098	/** @todo consider which extra parameters would be helpful for each probe. */
12099	case DBGFEVENT_END: break;
12100	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12101	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12102	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12103	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12104	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12105	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12106	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12107	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12108	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12109	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12110	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12111	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12112	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12113	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12114	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12115	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12116	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12117	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12118	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12119	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12120	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12121	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12122	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12123	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12124	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12125	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12126	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12127	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12128	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12129	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12130	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12131	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12132	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12133	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12134	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12135	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12136	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12137	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12138	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12139	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12140	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12141	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12142	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12143	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12144	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12145	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12146	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12147	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12148	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12149	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12150	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12151	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12152	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12153	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12154	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12155	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12156	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12157	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12158	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12159	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12160	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12161	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12162	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12163	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12164	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12165	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12166	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12167	}
12168	switch (enmEvent2)
12169	{
12170	/** @todo consider which extra parameters would be helpful for each probe. */
12171	case DBGFEVENT_END: break;
12172	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12173	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12174	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12175	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12176	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12177	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12178	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12179	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12180	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12181	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12182	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12183	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12184	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12185	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12186	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12187	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12188	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12189	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12190	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12191	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12192	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12193	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12194	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12195	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12196	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12197	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12198	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12199	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12200	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12201	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12202	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12203	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12204	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12205	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12206	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12207	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12208	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12209	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12210	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12211	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12212	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12213	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12214	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12215	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12216	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12217	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12218	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12219	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12220	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12221	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12222	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12223	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12224	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12225	}
12226	}
12227
12228	/*
12229	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12230	* the DBGF call will do a full check).
12231	*
12232	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12233	* Note! If we have to events, we prioritize the first, i.e. the instruction
12234	* one, in order to avoid event nesting.
12235	*/
12236	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
12237	if ( enmEvent1 != DBGFEVENT_END
12238	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12239	{
12240	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12241	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12242	if (rcStrict != VINF_SUCCESS)
12243	return rcStrict;
12244	}
12245	else if ( enmEvent2 != DBGFEVENT_END
12246	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12247	{
12248	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12249	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12250	if (rcStrict != VINF_SUCCESS)
12251	return rcStrict;
12252	}
12253
12254	return VINF_SUCCESS;
12255	}
12256
12257
12258	/**
12259	* Single-stepping VM-exit filtering.
12260	*
12261	* This is preprocessing the VM-exits and deciding whether we've gotten far
12262	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12263	* handling is performed.
12264	*
12265	* @returns Strict VBox status code (i.e. informational status codes too).
12266	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12267	* @param pVmxTransient The VMX-transient structure.
12268	* @param pDbgState The debug state.
12269	*/
12270	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12271	{
12272	/*
12273	* Expensive (saves context) generic dtrace VM-exit probe.
12274	*/
12275	uint32_t const uExitReason = pVmxTransient->uExitReason;
12276	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12277	{ /* more likely */ }
12278	else
12279	{
12280	hmR0VmxReadExitQualVmcs(pVmxTransient);
12281	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12282	AssertRC(rc);
12283	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12284	}
12285
12286	/*
12287	* Check for host NMI, just to get that out of the way.
12288	*/
12289	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12290	{ /* normally likely */ }
12291	else
12292	{
12293	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12294	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12295	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12296	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12297	}
12298
12299	/*
12300	* Check for single stepping event if we're stepping.
12301	*/
12302	if (pVCpu->hm.s.fSingleInstruction)
12303	{
12304	switch (uExitReason)
12305	{
12306	case VMX_EXIT_MTF:
12307	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12308
12309	/* Various events: */
12310	case VMX_EXIT_XCPT_OR_NMI:
12311	case VMX_EXIT_EXT_INT:
12312	case VMX_EXIT_TRIPLE_FAULT:
12313	case VMX_EXIT_INT_WINDOW:
12314	case VMX_EXIT_NMI_WINDOW:
12315	case VMX_EXIT_TASK_SWITCH:
12316	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12317	case VMX_EXIT_APIC_ACCESS:
12318	case VMX_EXIT_EPT_VIOLATION:
12319	case VMX_EXIT_EPT_MISCONFIG:
12320	case VMX_EXIT_PREEMPT_TIMER:
12321
12322	/* Instruction specific VM-exits: */
12323	case VMX_EXIT_CPUID:
12324	case VMX_EXIT_GETSEC:
12325	case VMX_EXIT_HLT:
12326	case VMX_EXIT_INVD:
12327	case VMX_EXIT_INVLPG:
12328	case VMX_EXIT_RDPMC:
12329	case VMX_EXIT_RDTSC:
12330	case VMX_EXIT_RSM:
12331	case VMX_EXIT_VMCALL:
12332	case VMX_EXIT_VMCLEAR:
12333	case VMX_EXIT_VMLAUNCH:
12334	case VMX_EXIT_VMPTRLD:
12335	case VMX_EXIT_VMPTRST:
12336	case VMX_EXIT_VMREAD:
12337	case VMX_EXIT_VMRESUME:
12338	case VMX_EXIT_VMWRITE:
12339	case VMX_EXIT_VMXOFF:
12340	case VMX_EXIT_VMXON:
12341	case VMX_EXIT_MOV_CRX:
12342	case VMX_EXIT_MOV_DRX:
12343	case VMX_EXIT_IO_INSTR:
12344	case VMX_EXIT_RDMSR:
12345	case VMX_EXIT_WRMSR:
12346	case VMX_EXIT_MWAIT:
12347	case VMX_EXIT_MONITOR:
12348	case VMX_EXIT_PAUSE:
12349	case VMX_EXIT_GDTR_IDTR_ACCESS:
12350	case VMX_EXIT_LDTR_TR_ACCESS:
12351	case VMX_EXIT_INVEPT:
12352	case VMX_EXIT_RDTSCP:
12353	case VMX_EXIT_INVVPID:
12354	case VMX_EXIT_WBINVD:
12355	case VMX_EXIT_XSETBV:
12356	case VMX_EXIT_RDRAND:
12357	case VMX_EXIT_INVPCID:
12358	case VMX_EXIT_VMFUNC:
12359	case VMX_EXIT_RDSEED:
12360	case VMX_EXIT_XSAVES:
12361	case VMX_EXIT_XRSTORS:
12362	{
12363	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12364	AssertRCReturn(rc, rc);
12365	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12366	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12367	return VINF_EM_DBG_STEPPED;
12368	break;
12369	}
12370
12371	/* Errors and unexpected events: */
12372	case VMX_EXIT_INIT_SIGNAL:
12373	case VMX_EXIT_SIPI:
12374	case VMX_EXIT_IO_SMI:
12375	case VMX_EXIT_SMI:
12376	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12377	case VMX_EXIT_ERR_MSR_LOAD:
12378	case VMX_EXIT_ERR_MACHINE_CHECK:
12379	case VMX_EXIT_PML_FULL:
12380	case VMX_EXIT_VIRTUALIZED_EOI:
12381	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12382	break;
12383
12384	default:
12385	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12386	break;
12387	}
12388	}
12389
12390	/*
12391	* Check for debugger event breakpoints and dtrace probes.
12392	*/
12393	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12394	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12395	{
12396	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12397	if (rcStrict != VINF_SUCCESS)
12398	return rcStrict;
12399	}
12400
12401	/*
12402	* Normal processing.
12403	*/
12404	#ifdef HMVMX_USE_FUNCTION_TABLE
12405	return g_aVMExitHandlers[uExitReason].pfn(pVCpu, pVmxTransient);
12406	#else
12407	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12408	#endif
12409	}
12410
12411
12412	/**
12413	* Single steps guest code using hardware-assisted VMX.
12414	*
12415	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12416	* but single-stepping through the hypervisor debugger.
12417	*
12418	* @returns Strict VBox status code (i.e. informational status codes too).
12419	* @param pVCpu The cross context virtual CPU structure.
12420	* @param pcLoops Pointer to the number of executed loops.
12421	*
12422	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12423	*/
12424	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
12425	{
12426	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12427	Assert(pcLoops);
12428	Assert(*pcLoops <= cMaxResumeLoops);
12429
12430	VMXTRANSIENT VmxTransient;
12431	RT_ZERO(VmxTransient);
12432	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12433
12434	/* Set HMCPU indicators. */
12435	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12436	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12437	pVCpu->hm.s.fDebugWantRdTscExit = false;
12438	pVCpu->hm.s.fUsingDebugLoop = true;
12439
12440	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12441	VMXRUNDBGSTATE DbgState;
12442	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12443	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12444
12445	/*
12446	* The loop.
12447	*/
12448	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12449	for (;;)
12450	{
12451	Assert(!HMR0SuspendPending());
12452	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12453	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12454	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12455
12456	/* Set up VM-execution controls the next two can respond to. */
12457	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12458
12459	/*
12460	* Preparatory work for running guest code, this may force us to
12461	* return to ring-3.
12462	*
12463	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12464	*/
12465	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12466	if (rcStrict != VINF_SUCCESS)
12467	break;
12468
12469	/* Interrupts are disabled at this point! */
12470	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12471
12472	/* Override any obnoxious code in the above two calls. */
12473	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12474
12475	/*
12476	* Finally execute the guest.
12477	*/
12478	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12479
12480	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12481	/* Interrupts are re-enabled at this point! */
12482
12483	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12484	if (RT_SUCCESS(rcRun))
12485	{ /* very likely */ }
12486	else
12487	{
12488	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12489	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12490	return rcRun;
12491	}
12492
12493	/* Profile the VM-exit. */
12494	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12496	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12497	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12498	HMVMX_START_EXIT_DISPATCH_PROF();
12499
12500	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12501
12502	/*
12503	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12504	*/
12505	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12506	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12507	if (rcStrict != VINF_SUCCESS)
12508	break;
12509	if (++(*pcLoops) > cMaxResumeLoops)
12510	{
12511	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12512	rcStrict = VINF_EM_RAW_INTERRUPT;
12513	break;
12514	}
12515
12516	/*
12517	* Stepping: Did the RIP change, if so, consider it a single step.
12518	* Otherwise, make sure one of the TFs gets set.
12519	*/
12520	if (fStepping)
12521	{
12522	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12523	AssertRC(rc);
12524	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12525	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12526	{
12527	rcStrict = VINF_EM_DBG_STEPPED;
12528	break;
12529	}
12530	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12531	}
12532
12533	/*
12534	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12535	*/
12536	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12537	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12538
12539	/* Restore all controls applied by hmR0VmxPreRunGuestDebugStateApply above. */
12540	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12541	Assert(rcStrict == VINF_SUCCESS);
12542	}
12543
12544	/*
12545	* Clear the X86_EFL_TF if necessary.
12546	*/
12547	if (pVCpu->hm.s.fClearTrapFlag)
12548	{
12549	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12550	AssertRC(rc);
12551	pVCpu->hm.s.fClearTrapFlag = false;
12552	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12553	}
12554	/** @todo there seems to be issues with the resume flag when the monitor trap
12555	* flag is pending without being used. Seen early in bios init when
12556	* accessing APIC page in protected mode. */
12557
12558	/* Restore HMCPU indicators. */
12559	pVCpu->hm.s.fUsingDebugLoop = false;
12560	pVCpu->hm.s.fDebugWantRdTscExit = false;
12561	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12562
12563	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12564	return rcStrict;
12565	}
12566
12567
12568	/** @} */
12569
12570
12571	/**
12572	* Checks if any expensive dtrace probes are enabled and we should go to the
12573	* debug loop.
12574	*
12575	* @returns true if we should use debug loop, false if not.
12576	*/
12577	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12578	{
12579	/* It's probably faster to OR the raw 32-bit counter variables together.
12580	Since the variables are in an array and the probes are next to one
12581	another (more or less), we have good locality. So, better read
12582	eight-nine cache lines ever time and only have one conditional, than
12583	128+ conditionals, right? */
12584	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12585	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12586	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12587	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12588	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12589	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12590	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12591	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12592	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12593	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12594	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12595	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12596	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12597	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12598	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12599	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12600	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12601	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12602	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12603	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12604	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12605	) != 0
12606	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12607	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12608	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12609	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12610	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12611	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12612	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12613	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12614	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12615	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12616	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12617	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12618	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12619	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12620	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12621	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12622	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12623	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12624	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12625	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12626	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12627	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12628	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12629	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12630	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12631	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12632	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12633	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12634	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12635	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12636	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12637	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12638	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12639	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12640	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12641	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12642	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12643	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12644	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12645	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12646	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12647	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12648	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12649	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12650	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12651	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12652	) != 0
12653	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12654	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12655	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12656	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12657	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12658	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12659	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12660	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12661	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12662	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12663	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12664	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12665	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12666	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12667	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12668	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12669	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12670	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12671	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12672	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12673	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12674	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12675	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12676	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12677	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12678	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12679	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12680	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12681	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12682	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12683	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12684	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12685	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12686	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12687	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12688	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12689	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12690	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12691	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12692	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12693	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12694	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12695	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12696	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12697	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12698	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12699	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12700	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12701	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12702	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12703	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12704	) != 0;
12705	}
12706
12707
12708	/**
12709	* Runs the guest using hardware-assisted VMX.
12710	*
12711	* @returns Strict VBox status code (i.e. informational status codes too).
12712	* @param pVCpu The cross context virtual CPU structure.
12713	*/
12714	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu)
12715	{
12716	AssertPtr(pVCpu);
12717	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12718	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12719	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12720	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12721
12722	VBOXSTRICTRC rcStrict;
12723	uint32_t cLoops = 0;
12724	for (;;)
12725	{
12726	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12727	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12728	#else
12729	NOREF(pCtx);
12730	bool const fInNestedGuestMode = false;
12731	#endif
12732	if (!fInNestedGuestMode)
12733	{
12734	if ( !pVCpu->hm.s.fUseDebugLoop
12735	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12736	&& !DBGFIsStepping(pVCpu)
12737	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12738	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12739	else
12740	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12741	}
12742	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12743	else
12744	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12745
12746	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12747	{
12748	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12749	continue;
12750	}
12751	if (rcStrict == VINF_VMX_VMEXIT)
12752	{
12753	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12754	continue;
12755	}
12756	#endif
12757	break;
12758	}
12759
12760	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12761	switch (rcLoop)
12762	{
12763	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12764	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12765	}
12766
12767	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12768	if (RT_FAILURE(rc2))
12769	{
12770	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12771	rcStrict = rc2;
12772	}
12773	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12774	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12775	return rcStrict;
12776	}
12777
12778
12779	#ifndef HMVMX_USE_FUNCTION_TABLE
12780	/**
12781	* Handles a guest VM-exit from hardware-assisted VMX execution.
12782	*
12783	* @returns Strict VBox status code (i.e. informational status codes too).
12784	* @param pVCpu The cross context virtual CPU structure.
12785	* @param pVmxTransient The VMX-transient structure.
12786	*/
12787	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12788	{
12789	#ifdef DEBUG_ramshankar
12790	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12791	do { \
12792	if (a_fSave != 0) \
12793	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12794	VBOXSTRICTRC rcStrict = a_CallExpr; \
12795	if (a_fSave != 0) \
12796	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12797	return rcStrict; \
12798	} while (0)
12799	#else
12800	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12801	#endif
12802	uint32_t const uExitReason = pVmxTransient->uExitReason;
12803	switch (uExitReason)
12804	{
12805	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12806	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12807	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12808	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12809	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12810	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12811	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12812	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12813	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12814	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12815	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12816	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12817	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12818	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12819	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12820	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12821	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12822	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12823	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12824	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12825	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12826	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12827	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12828	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12829	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12830	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12831	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12832	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12833	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12834	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12835	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12836	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12837	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12838	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12839	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12840	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12841	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12842	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12843	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12844	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12845	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12846	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12847	#else
12848	case VMX_EXIT_VMCLEAR:
12849	case VMX_EXIT_VMLAUNCH:
12850	case VMX_EXIT_VMPTRLD:
12851	case VMX_EXIT_VMPTRST:
12852	case VMX_EXIT_VMREAD:
12853	case VMX_EXIT_VMRESUME:
12854	case VMX_EXIT_VMWRITE:
12855	case VMX_EXIT_VMXOFF:
12856	case VMX_EXIT_VMXON:
12857	case VMX_EXIT_INVVPID:
12858	case VMX_EXIT_INVEPT:
12859	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12860	#endif
12861
12862	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12863	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12864	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12865
12866	case VMX_EXIT_INIT_SIGNAL:
12867	case VMX_EXIT_SIPI:
12868	case VMX_EXIT_IO_SMI:
12869	case VMX_EXIT_SMI:
12870	case VMX_EXIT_ERR_MSR_LOAD:
12871	case VMX_EXIT_ERR_MACHINE_CHECK:
12872	case VMX_EXIT_PML_FULL:
12873	case VMX_EXIT_VIRTUALIZED_EOI:
12874	case VMX_EXIT_GDTR_IDTR_ACCESS:
12875	case VMX_EXIT_LDTR_TR_ACCESS:
12876	case VMX_EXIT_APIC_WRITE:
12877	case VMX_EXIT_RDRAND:
12878	case VMX_EXIT_RSM:
12879	case VMX_EXIT_VMFUNC:
12880	case VMX_EXIT_ENCLS:
12881	case VMX_EXIT_RDSEED:
12882	case VMX_EXIT_XSAVES:
12883	case VMX_EXIT_XRSTORS:
12884	case VMX_EXIT_UMWAIT:
12885	case VMX_EXIT_TPAUSE:
12886	default:
12887	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12888	}
12889	#undef VMEXIT_CALL_RET
12890	}
12891	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12892
12893
12894	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12895	/**
12896	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12897	*
12898	* @returns Strict VBox status code (i.e. informational status codes too).
12899	* @param pVCpu The cross context virtual CPU structure.
12900	* @param pVmxTransient The VMX-transient structure.
12901	*/
12902	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12903	{
12904	uint32_t const uExitReason = pVmxTransient->uExitReason;
12905	switch (uExitReason)
12906	{
12907	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12908	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12909	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12910	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12911	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12912
12913	/*
12914	* We shouldn't direct host physical interrupts to the nested-guest.
12915	*/
12916	case VMX_EXIT_EXT_INT:
12917	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12918
12919	/*
12920	* Instructions that cause VM-exits unconditionally or the condition is
12921	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12922	* happens, it's guaranteed to be a nested-guest VM-exit).
12923	*
12924	* - Provides VM-exit instruction length ONLY.
12925	*/
12926	case VMX_EXIT_CPUID: /* Unconditional. */
12927	case VMX_EXIT_VMCALL:
12928	case VMX_EXIT_GETSEC:
12929	case VMX_EXIT_INVD:
12930	case VMX_EXIT_XSETBV:
12931	case VMX_EXIT_VMLAUNCH:
12932	case VMX_EXIT_VMRESUME:
12933	case VMX_EXIT_VMXOFF:
12934	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
12935	case VMX_EXIT_VMFUNC:
12936	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12937
12938	/*
12939	* Instructions that cause VM-exits unconditionally or the condition is
12940	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12941	* happens, it's guaranteed to be a nested-guest VM-exit).
12942	*
12943	* - Provides VM-exit instruction length.
12944	* - Provides VM-exit information.
12945	* - Optionally provides Exit qualification.
12946	*
12947	* Since Exit qualification is 0 for all VM-exits where it is not
12948	* applicable, reading and passing it to the guest should produce
12949	* defined behavior.
12950	*
12951	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12952	*/
12953	case VMX_EXIT_INVEPT: /* Unconditional. */
12954	case VMX_EXIT_INVVPID:
12955	case VMX_EXIT_VMCLEAR:
12956	case VMX_EXIT_VMPTRLD:
12957	case VMX_EXIT_VMPTRST:
12958	case VMX_EXIT_VMXON:
12959	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
12960	case VMX_EXIT_LDTR_TR_ACCESS:
12961	case VMX_EXIT_RDRAND:
12962	case VMX_EXIT_RDSEED:
12963	case VMX_EXIT_XSAVES:
12964	case VMX_EXIT_XRSTORS:
12965	case VMX_EXIT_UMWAIT:
12966	case VMX_EXIT_TPAUSE:
12967	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12968
12969	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12970	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12971	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12972	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12973	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12974	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12975	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12976	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12977	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12978	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12979	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12980	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12981	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12982	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12983	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12984	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12985	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12986	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12987	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12988
12989	case VMX_EXIT_PREEMPT_TIMER:
12990	{
12991	/** @todo NSTVMX: Preempt timer. */
12992	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12993	}
12994
12995	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12996	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12997
12998	case VMX_EXIT_VMREAD:
12999	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13000
13001	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13002	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13003
13004	case VMX_EXIT_INIT_SIGNAL:
13005	case VMX_EXIT_SIPI:
13006	case VMX_EXIT_IO_SMI:
13007	case VMX_EXIT_SMI:
13008	case VMX_EXIT_ERR_MSR_LOAD:
13009	case VMX_EXIT_ERR_MACHINE_CHECK:
13010	case VMX_EXIT_PML_FULL:
13011	case VMX_EXIT_RSM:
13012	default:
13013	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13014	}
13015	}
13016	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13017
13018
13019	/** @name VM-exit helpers.
13020	* @{
13021	*/
13022	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13023	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13024	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13025
13026	/** Macro for VM-exits called unexpectedly. */
13027	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
13028	do { \
13029	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
13030	return VERR_VMX_UNEXPECTED_EXIT; \
13031	} while (0)
13032
13033	#ifdef VBOX_STRICT
13034	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13035	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13036	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13037
13038	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13039	do { \
13040	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13041	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13042	} while (0)
13043
13044	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13045	do { \
13046	AssertPtr((a_pVCpu)); \
13047	AssertPtr((a_pVmxTransient)); \
13048	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13049	Assert((a_pVmxTransient)->pVmcsInfo); \
13050	Assert(ASMIntAreEnabled()); \
13051	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13052	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13053	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
13054	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13055	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13056	HMVMX_ASSERT_PREEMPT_CPUID(); \
13057	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13058	} while (0)
13059
13060	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13061	do { \
13062	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13063	Assert((a_pVmxTransient)->fIsNestedGuest); \
13064	} while (0)
13065
13066	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13067	do { \
13068	Log4Func(("\n")); \
13069	} while (0)
13070	#else
13071	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13072	do { \
13073	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13074	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13075	} while (0)
13076
13077	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13078	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13079
13080	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13081	#endif
13082
13083	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13084	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13085	* guests that attempted to execute a VMX instruction. */
13086	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13087	do \
13088	{ \
13089	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13090	if (rcStrictTmp == VINF_SUCCESS) \
13091	{ /* likely */ } \
13092	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13093	{ \
13094	Assert((a_pVCpu)->hm.s.Event.fPending); \
13095	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13096	return VINF_SUCCESS; \
13097	} \
13098	else \
13099	{ \
13100	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13101	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13102	} \
13103	} while (0)
13104
13105	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13106	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13107	do \
13108	{ \
13109	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13110	(a_pGCPtrEffAddr)); \
13111	if (rcStrictTmp == VINF_SUCCESS) \
13112	{ /* likely */ } \
13113	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13114	{ \
13115	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13116	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13117	NOREF(uXcptTmp); \
13118	return VINF_SUCCESS; \
13119	} \
13120	else \
13121	{ \
13122	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13123	return rcStrictTmp; \
13124	} \
13125	} while (0)
13126	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13127
13128
13129	/**
13130	* Advances the guest RIP by the specified number of bytes.
13131	*
13132	* @param pVCpu The cross context virtual CPU structure.
13133	* @param cbInstr Number of bytes to advance the RIP by.
13134	*
13135	* @remarks No-long-jump zone!!!
13136	*/
13137	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
13138	{
13139	/* Advance the RIP. */
13140	pVCpu->cpum.GstCtx.rip += cbInstr;
13141	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13142
13143	/* Update interrupt inhibition. */
13144	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13145	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13146	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13147	}
13148
13149
13150	/**
13151	* Advances the guest RIP after reading it from the VMCS.
13152	*
13153	* @returns VBox status code, no informational status codes.
13154	* @param pVCpu The cross context virtual CPU structure.
13155	* @param pVmxTransient The VMX-transient structure.
13156	*
13157	* @remarks No-long-jump zone!!!
13158	*/
13159	static int hmR0VmxAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13160	{
13161	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13162	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13163	AssertRCReturn(rc, rc);
13164
13165	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
13166	return VINF_SUCCESS;
13167	}
13168
13169
13170	/**
13171	* Handle a condition that occurred while delivering an event through the guest or
13172	* nested-guest IDT.
13173	*
13174	* @returns Strict VBox status code (i.e. informational status codes too).
13175	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13176	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13177	* to continue execution of the guest which will delivery the \#DF.
13178	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13179	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13180	*
13181	* @param pVCpu The cross context virtual CPU structure.
13182	* @param pVmxTransient The VMX-transient structure.
13183	*
13184	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13185	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
13186	* is due to an EPT violation, PML full or SPP-related event.
13187	*
13188	* @remarks No-long-jump zone!!!
13189	*/
13190	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13191	{
13192	Assert(!pVCpu->hm.s.Event.fPending);
13193	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
13194	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13195	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13196	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13197	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
13198
13199	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13200	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13201	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
13202	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13203	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
13204	{
13205	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
13206	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
13207
13208	/*
13209	* If the event was a software interrupt (generated with INT n) or a software exception
13210	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13211	* can handle the VM-exit and continue guest execution which will re-execute the
13212	* instruction rather than re-injecting the exception, as that can cause premature
13213	* trips to ring-3 before injection and involve TRPM which currently has no way of
13214	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13215	* the problem).
13216	*/
13217	IEMXCPTRAISE enmRaise;
13218	IEMXCPTRAISEINFO fRaiseInfo;
13219	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13220	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13221	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13222	{
13223	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13224	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13225	}
13226	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
13227	{
13228	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13229	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13230	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
13231
13232	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13233	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13234
13235	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13236
13237	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13238	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13239	{
13240	pVmxTransient->fVectoringPF = true;
13241	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13242	}
13243	}
13244	else
13245	{
13246	/*
13247	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13248	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13249	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13250	*/
13251	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13252	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13253	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13254	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13255	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13256	}
13257
13258	/*
13259	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13260	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13261	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13262	* subsequent VM-entry would fail, see @bugref{7445}.
13263	*
13264	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13265	*/
13266	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13267	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
13268	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13269	&& CPUMIsGuestNmiBlocking(pVCpu))
13270	{
13271	CPUMSetGuestNmiBlocking(pVCpu, false);
13272	}
13273
13274	switch (enmRaise)
13275	{
13276	case IEMXCPTRAISE_CURRENT_XCPT:
13277	{
13278	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
13279	Assert(rcStrict == VINF_SUCCESS);
13280	break;
13281	}
13282
13283	case IEMXCPTRAISE_PREV_EVENT:
13284	{
13285	uint32_t u32ErrCode;
13286	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13287	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13288	else
13289	u32ErrCode = 0;
13290
13291	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13292	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13293	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13294	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13295
13296	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13297	pVCpu->hm.s.Event.u32ErrCode));
13298	Assert(rcStrict == VINF_SUCCESS);
13299	break;
13300	}
13301
13302	case IEMXCPTRAISE_REEXEC_INSTR:
13303	Assert(rcStrict == VINF_SUCCESS);
13304	break;
13305
13306	case IEMXCPTRAISE_DOUBLE_FAULT:
13307	{
13308	/*
13309	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13310	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13311	*/
13312	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13313	{
13314	pVmxTransient->fVectoringDoublePF = true;
13315	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13316	pVCpu->cpum.GstCtx.cr2));
13317	rcStrict = VINF_SUCCESS;
13318	}
13319	else
13320	{
13321	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13322	hmR0VmxSetPendingXcptDF(pVCpu);
13323	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13324	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13325	rcStrict = VINF_HM_DOUBLE_FAULT;
13326	}
13327	break;
13328	}
13329
13330	case IEMXCPTRAISE_TRIPLE_FAULT:
13331	{
13332	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13333	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13334	rcStrict = VINF_EM_RESET;
13335	break;
13336	}
13337
13338	case IEMXCPTRAISE_CPU_HANG:
13339	{
13340	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13341	rcStrict = VERR_EM_GUEST_CPU_HANG;
13342	break;
13343	}
13344
13345	default:
13346	{
13347	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13348	rcStrict = VERR_VMX_IPE_2;
13349	break;
13350	}
13351	}
13352	}
13353	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13354	&& !CPUMIsGuestNmiBlocking(pVCpu))
13355	{
13356	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13357	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13358	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13359	{
13360	/*
13361	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13362	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13363	* that virtual NMIs remain blocked until the IRET execution is completed.
13364	*
13365	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13366	*/
13367	CPUMSetGuestNmiBlocking(pVCpu, true);
13368	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13369	}
13370	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13371	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13372	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13373	{
13374	/*
13375	* Execution of IRET caused an EPT violation, page-modification log-full event or
13376	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13377	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13378	* that virtual NMIs remain blocked until the IRET execution is completed.
13379	*
13380	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13381	*/
13382	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13383	{
13384	CPUMSetGuestNmiBlocking(pVCpu, true);
13385	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13386	}
13387	}
13388	}
13389
13390	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13391	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13392	return rcStrict;
13393	}
13394
13395
13396	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13397	/**
13398	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13399	* guest attempting to execute a VMX instruction.
13400	*
13401	* @returns Strict VBox status code (i.e. informational status codes too).
13402	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13403	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13404	*
13405	* @param pVCpu The cross context virtual CPU structure.
13406	* @param uExitReason The VM-exit reason.
13407	*
13408	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13409	* @remarks No-long-jump zone!!!
13410	*/
13411	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
13412	{
13413	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13414	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13415
13416	/*
13417	* The physical CPU would have already checked the CPU mode/code segment.
13418	* We shall just assert here for paranoia.
13419	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13420	*/
13421	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13422	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13423	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13424
13425	if (uExitReason == VMX_EXIT_VMXON)
13426	{
13427	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13428
13429	/*
13430	* We check CR4.VMXE because it is required to be always set while in VMX operation
13431	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13432	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13433	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13434	*/
13435	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13436	{
13437	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13438	hmR0VmxSetPendingXcptUD(pVCpu);
13439	return VINF_HM_PENDING_XCPT;
13440	}
13441	}
13442	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13443	{
13444	/*
13445	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13446	* (other than VMXON), we need to raise a #UD.
13447	*/
13448	Log4Func(("Not in VMX root mode -> #UD\n"));
13449	hmR0VmxSetPendingXcptUD(pVCpu);
13450	return VINF_HM_PENDING_XCPT;
13451	}
13452
13453	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13454	return VINF_SUCCESS;
13455	}
13456
13457
13458	/**
13459	* Decodes the memory operand of an instruction that caused a VM-exit.
13460	*
13461	* The Exit qualification field provides the displacement field for memory
13462	* operand instructions, if any.
13463	*
13464	* @returns Strict VBox status code (i.e. informational status codes too).
13465	* @retval VINF_SUCCESS if the operand was successfully decoded.
13466	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13467	* operand.
13468	* @param pVCpu The cross context virtual CPU structure.
13469	* @param uExitInstrInfo The VM-exit instruction information field.
13470	* @param enmMemAccess The memory operand's access type (read or write).
13471	* @param GCPtrDisp The instruction displacement field, if any. For
13472	* RIP-relative addressing pass RIP + displacement here.
13473	* @param pGCPtrMem Where to store the effective destination memory address.
13474	*
13475	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13476	* virtual-8086 mode hence skips those checks while verifying if the
13477	* segment is valid.
13478	*/
13479	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13480	PRTGCPTR pGCPtrMem)
13481	{
13482	Assert(pGCPtrMem);
13483	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13484	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13485	\| CPUMCTX_EXTRN_CR0);
13486
13487	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13488	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13489	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13490
13491	VMXEXITINSTRINFO ExitInstrInfo;
13492	ExitInstrInfo.u = uExitInstrInfo;
13493	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13494	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13495	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13496	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13497	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13498	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13499	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13500	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13501	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13502
13503	/*
13504	* Validate instruction information.
13505	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13506	*/
13507	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13508	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13509	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13510	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13511	AssertLogRelMsgReturn(fIsMemOperand,
13512	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13513
13514	/*
13515	* Compute the complete effective address.
13516	*
13517	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13518	* See AMD spec. 4.5.2 "Segment Registers".
13519	*/
13520	RTGCPTR GCPtrMem = GCPtrDisp;
13521	if (fBaseRegValid)
13522	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13523	if (fIdxRegValid)
13524	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13525
13526	RTGCPTR const GCPtrOff = GCPtrMem;
13527	if ( !fIsLongMode
13528	\|\| iSegReg >= X86_SREG_FS)
13529	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13530	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13531
13532	/*
13533	* Validate effective address.
13534	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13535	*/
13536	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13537	Assert(cbAccess > 0);
13538	if (fIsLongMode)
13539	{
13540	if (X86_IS_CANONICAL(GCPtrMem))
13541	{
13542	*pGCPtrMem = GCPtrMem;
13543	return VINF_SUCCESS;
13544	}
13545
13546	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13547	* "Data Limit Checks in 64-bit Mode". */
13548	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13549	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13550	return VINF_HM_PENDING_XCPT;
13551	}
13552
13553	/*
13554	* This is a watered down version of iemMemApplySegment().
13555	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13556	* and segment CPL/DPL checks are skipped.
13557	*/
13558	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13559	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13560	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13561
13562	/* Check if the segment is present and usable. */
13563	if ( pSel->Attr.n.u1Present
13564	&& !pSel->Attr.n.u1Unusable)
13565	{
13566	Assert(pSel->Attr.n.u1DescType);
13567	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13568	{
13569	/* Check permissions for the data segment. */
13570	if ( enmMemAccess == VMXMEMACCESS_WRITE
13571	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13572	{
13573	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13574	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13575	return VINF_HM_PENDING_XCPT;
13576	}
13577
13578	/* Check limits if it's a normal data segment. */
13579	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13580	{
13581	if ( GCPtrFirst32 > pSel->u32Limit
13582	\|\| GCPtrLast32 > pSel->u32Limit)
13583	{
13584	Log4Func(("Data segment limit exceeded. "
13585	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13586	GCPtrLast32, pSel->u32Limit));
13587	if (iSegReg == X86_SREG_SS)
13588	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13589	else
13590	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13591	return VINF_HM_PENDING_XCPT;
13592	}
13593	}
13594	else
13595	{
13596	/* Check limits if it's an expand-down data segment.
13597	Note! The upper boundary is defined by the B bit, not the G bit! */
13598	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13599	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13600	{
13601	Log4Func(("Expand-down data segment limit exceeded. "
13602	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13603	GCPtrLast32, pSel->u32Limit));
13604	if (iSegReg == X86_SREG_SS)
13605	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13606	else
13607	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13608	return VINF_HM_PENDING_XCPT;
13609	}
13610	}
13611	}
13612	else
13613	{
13614	/* Check permissions for the code segment. */
13615	if ( enmMemAccess == VMXMEMACCESS_WRITE
13616	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13617	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13618	{
13619	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13620	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13621	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13622	return VINF_HM_PENDING_XCPT;
13623	}
13624
13625	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13626	if ( GCPtrFirst32 > pSel->u32Limit
13627	\|\| GCPtrLast32 > pSel->u32Limit)
13628	{
13629	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13630	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13631	if (iSegReg == X86_SREG_SS)
13632	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13633	else
13634	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13635	return VINF_HM_PENDING_XCPT;
13636	}
13637	}
13638	}
13639	else
13640	{
13641	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13642	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13643	return VINF_HM_PENDING_XCPT;
13644	}
13645
13646	*pGCPtrMem = GCPtrMem;
13647	return VINF_SUCCESS;
13648	}
13649	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13650
13651
13652	/**
13653	* VM-exit helper for LMSW.
13654	*/
13655	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13656	{
13657	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13658	AssertRCReturn(rc, rc);
13659
13660	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13661	AssertMsg( rcStrict == VINF_SUCCESS
13662	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13663
13664	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13665	if (rcStrict == VINF_IEM_RAISED_XCPT)
13666	{
13667	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13668	rcStrict = VINF_SUCCESS;
13669	}
13670
13671	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13672	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13673	return rcStrict;
13674	}
13675
13676
13677	/**
13678	* VM-exit helper for CLTS.
13679	*/
13680	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13681	{
13682	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13683	AssertRCReturn(rc, rc);
13684
13685	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13686	AssertMsg( rcStrict == VINF_SUCCESS
13687	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13688
13689	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13690	if (rcStrict == VINF_IEM_RAISED_XCPT)
13691	{
13692	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13693	rcStrict = VINF_SUCCESS;
13694	}
13695
13696	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13697	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13698	return rcStrict;
13699	}
13700
13701
13702	/**
13703	* VM-exit helper for MOV from CRx (CRx read).
13704	*/
13705	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13706	{
13707	Assert(iCrReg < 16);
13708	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13709
13710	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13711	AssertRCReturn(rc, rc);
13712
13713	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13714	AssertMsg( rcStrict == VINF_SUCCESS
13715	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13716
13717	if (iGReg == X86_GREG_xSP)
13718	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13719	else
13720	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13721	#ifdef VBOX_WITH_STATISTICS
13722	switch (iCrReg)
13723	{
13724	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13725	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13726	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13727	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13728	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13729	}
13730	#endif
13731	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13732	return rcStrict;
13733	}
13734
13735
13736	/**
13737	* VM-exit helper for MOV to CRx (CRx write).
13738	*/
13739	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13740	{
13741	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13742	AssertRCReturn(rc, rc);
13743
13744	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13745	AssertMsg( rcStrict == VINF_SUCCESS
13746	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13747	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13748
13749	switch (iCrReg)
13750	{
13751	case 0:
13752	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
13753	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13754	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13755	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13756	break;
13757
13758	case 2:
13759	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13760	/* Nothing to do here, CR2 it's not part of the VMCS. */
13761	break;
13762
13763	case 3:
13764	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13765	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13766	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13767	break;
13768
13769	case 4:
13770	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13771	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13772	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13773	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13774	break;
13775
13776	case 8:
13777	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13778	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13779	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13780	break;
13781
13782	default:
13783	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13784	break;
13785	}
13786
13787	if (rcStrict == VINF_IEM_RAISED_XCPT)
13788	{
13789	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13790	rcStrict = VINF_SUCCESS;
13791	}
13792	return rcStrict;
13793	}
13794
13795
13796	/**
13797	* VM-exit exception handler for \#PF (Page-fault exception).
13798	*
13799	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13800	*/
13801	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13802	{
13803	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13804	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13805	hmR0VmxReadExitQualVmcs(pVmxTransient);
13806
13807	if (!pVM->hm.s.fNestedPaging)
13808	{ /* likely */ }
13809	else
13810	{
13811	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13812	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13813	#endif
13814	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13815	if (!pVmxTransient->fVectoringDoublePF)
13816	{
13817	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13818	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13819	}
13820	else
13821	{
13822	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13823	Assert(!pVmxTransient->fIsNestedGuest);
13824	hmR0VmxSetPendingXcptDF(pVCpu);
13825	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13826	}
13827	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13828	return VINF_SUCCESS;
13829	}
13830
13831	Assert(!pVmxTransient->fIsNestedGuest);
13832
13833	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13834	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13835	if (pVmxTransient->fVectoringPF)
13836	{
13837	Assert(pVCpu->hm.s.Event.fPending);
13838	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13839	}
13840
13841	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13842	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13843	AssertRCReturn(rc, rc);
13844
13845	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
13846	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
13847
13848	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13849	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13850
13851	Log4Func(("#PF: rc=%Rrc\n", rc));
13852	if (rc == VINF_SUCCESS)
13853	{
13854	/*
13855	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13856	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13857	*/
13858	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13859	TRPMResetTrap(pVCpu);
13860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13861	return rc;
13862	}
13863
13864	if (rc == VINF_EM_RAW_GUEST_TRAP)
13865	{
13866	if (!pVmxTransient->fVectoringDoublePF)
13867	{
13868	/* It's a guest page fault and needs to be reflected to the guest. */
13869	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
13870	TRPMResetTrap(pVCpu);
13871	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13872	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13873	uGstErrorCode, pVmxTransient->uExitQual);
13874	}
13875	else
13876	{
13877	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13878	TRPMResetTrap(pVCpu);
13879	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13880	hmR0VmxSetPendingXcptDF(pVCpu);
13881	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13882	}
13883
13884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13885	return VINF_SUCCESS;
13886	}
13887
13888	TRPMResetTrap(pVCpu);
13889	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13890	return rc;
13891	}
13892
13893
13894	/**
13895	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13896	*
13897	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13898	*/
13899	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13900	{
13901	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13902	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13903
13904	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13905	AssertRCReturn(rc, rc);
13906
13907	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13908	{
13909	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13910	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13911
13912	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13913	* provides VM-exit instruction length. If this causes problem later,
13914	* disassemble the instruction like it's done on AMD-V. */
13915	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13916	AssertRCReturn(rc2, rc2);
13917	return rc;
13918	}
13919
13920	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
13921	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13922	return VINF_SUCCESS;
13923	}
13924
13925
13926	/**
13927	* VM-exit exception handler for \#BP (Breakpoint exception).
13928	*
13929	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13930	*/
13931	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13932	{
13933	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13934	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13935
13936	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13937	AssertRCReturn(rc, rc);
13938
13939	if (!pVmxTransient->fIsNestedGuest)
13940	rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13941	else
13942	rc = VINF_EM_RAW_GUEST_TRAP;
13943
13944	if (rc == VINF_EM_RAW_GUEST_TRAP)
13945	{
13946	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13947	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13948	rc = VINF_SUCCESS;
13949	}
13950
13951	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
13952	return rc;
13953	}
13954
13955
13956	/**
13957	* VM-exit exception handler for \#AC (Alignment-check exception).
13958	*
13959	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13960	*/
13961	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13962	{
13963	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13964	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13965
13966	/* Re-inject it. We'll detect any nesting before getting here. */
13967	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13968	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13969	return VINF_SUCCESS;
13970	}
13971
13972
13973	/**
13974	* VM-exit exception handler for \#DB (Debug exception).
13975	*
13976	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13977	*/
13978	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13979	{
13980	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13981	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13982
13983	/*
13984	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13985	*/
13986	hmR0VmxReadExitQualVmcs(pVmxTransient);
13987
13988	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13989	uint64_t const uDR6 = X86_DR6_INIT_VAL
13990	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13991	\| X86_DR6_BD \| X86_DR6_BS));
13992
13993	int rc;
13994	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13995	if (!pVmxTransient->fIsNestedGuest)
13996	{
13997	rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13998
13999	/*
14000	* Prevents stepping twice over the same instruction when the guest is stepping using
14001	* EFLAGS.TF and the hypervisor debugger is stepping using MTF.
14002	* Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
14003	*/
14004	if ( rc == VINF_EM_DBG_STEPPED
14005	&& (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
14006	{
14007	Assert(pVCpu->hm.s.fSingleInstruction);
14008	rc = VINF_EM_RAW_GUEST_TRAP;
14009	}
14010	}
14011	else
14012	rc = VINF_EM_RAW_GUEST_TRAP;
14013	Log6Func(("rc=%Rrc\n", rc));
14014	if (rc == VINF_EM_RAW_GUEST_TRAP)
14015	{
14016	/*
14017	* The exception was for the guest. Update DR6, DR7.GD and
14018	* IA32_DEBUGCTL.LBR before forwarding it.
14019	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14020	*/
14021	VMMRZCallRing3Disable(pVCpu);
14022	HM_DISABLE_PREEMPT(pVCpu);
14023
14024	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14025	pCtx->dr[6] \|= uDR6;
14026	if (CPUMIsGuestDebugStateActive(pVCpu))
14027	ASMSetDR6(pCtx->dr[6]);
14028
14029	HM_RESTORE_PREEMPT();
14030	VMMRZCallRing3Enable(pVCpu);
14031
14032	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14033	AssertRCReturn(rc, rc);
14034
14035	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14036	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
14037
14038	/* Paranoia. */
14039	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
14040	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14041
14042	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
14043	AssertRC(rc);
14044
14045	/*
14046	* Raise #DB in the guest.
14047	*
14048	* It is important to reflect exactly what the VM-exit gave us (preserving the
14049	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14050	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14051	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14052	*
14053	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14054	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14055	*/
14056	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14057	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14058	return VINF_SUCCESS;
14059	}
14060
14061	/*
14062	* Not a guest trap, must be a hypervisor related debug event then.
14063	* Update DR6 in case someone is interested in it.
14064	*/
14065	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14066	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14067	CPUMSetHyperDR6(pVCpu, uDR6);
14068
14069	return rc;
14070	}
14071
14072
14073	/**
14074	* Hacks its way around the lovely mesa driver's backdoor accesses.
14075	*
14076	* @sa hmR0SvmHandleMesaDrvGp.
14077	*/
14078	static int hmR0VmxHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14079	{
14080	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14081	RT_NOREF(pCtx);
14082
14083	/* For now we'll just skip the instruction. */
14084	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14085	}
14086
14087
14088	/**
14089	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14090	* backdoor logging w/o checking what it is running inside.
14091	*
14092	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14093	* backdoor port and magic numbers loaded in registers.
14094	*
14095	* @returns true if it is, false if it isn't.
14096	* @sa hmR0SvmIsMesaDrvGp.
14097	*/
14098	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14099	{
14100	/* 0xed: IN eAX,dx */
14101	uint8_t abInstr[1];
14102	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
14103	return false;
14104
14105	/* Check that it is #GP(0). */
14106	if (pVmxTransient->uExitIntErrorCode != 0)
14107	return false;
14108
14109	/* Check magic and port. */
14110	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14111	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14112	if (pCtx->rax != UINT32_C(0x564d5868))
14113	return false;
14114	if (pCtx->dx != UINT32_C(0x5658))
14115	return false;
14116
14117	/* Flat ring-3 CS. */
14118	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14119	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14120	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14121	if (pCtx->cs.Attr.n.u2Dpl != 3)
14122	return false;
14123	if (pCtx->cs.u64Base != 0)
14124	return false;
14125
14126	/* Check opcode. */
14127	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14128	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14129	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14130	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14131	if (RT_FAILURE(rc))
14132	return false;
14133	if (abInstr[0] != 0xed)
14134	return false;
14135
14136	return true;
14137	}
14138
14139
14140	/**
14141	* VM-exit exception handler for \#GP (General-protection exception).
14142	*
14143	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
14144	*/
14145	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14146	{
14147	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14148	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14149
14150	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14151	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14152	if (pVmcsInfo->RealMode.fRealOnV86Active)
14153	{ /* likely */ }
14154	else
14155	{
14156	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14157	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14158	#endif
14159	/*
14160	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
14161	* executing a nested-guest, reflect #GP to the guest or nested-guest.
14162	*/
14163	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14164	AssertRCReturn(rc, rc);
14165	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14166	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14167
14168	if ( pVmxTransient->fIsNestedGuest
14169	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14170	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14171	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14172	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14173	else
14174	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14175	return rc;
14176	}
14177
14178	Assert(CPUMIsGuestInRealModeEx(pCtx));
14179	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14180	Assert(!pVmxTransient->fIsNestedGuest);
14181
14182	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14183	AssertRCReturn(rc, rc);
14184
14185	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14186	if (rcStrict == VINF_SUCCESS)
14187	{
14188	if (!CPUMIsGuestInRealModeEx(pCtx))
14189	{
14190	/*
14191	* The guest is no longer in real-mode, check if we can continue executing the
14192	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14193	*/
14194	pVmcsInfo->RealMode.fRealOnV86Active = false;
14195	if (HMCanExecuteVmxGuest(pVCpu->pVMR0, pVCpu, pCtx))
14196	{
14197	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14198	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14199	}
14200	else
14201	{
14202	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14203	rcStrict = VINF_EM_RESCHEDULE;
14204	}
14205	}
14206	else
14207	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14208	}
14209	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14210	{
14211	rcStrict = VINF_SUCCESS;
14212	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14213	}
14214	return VBOXSTRICTRC_VAL(rcStrict);
14215	}
14216
14217
14218	/**
14219	* VM-exit exception handler wrapper for all other exceptions that are not handled
14220	* by a specific handler.
14221	*
14222	* This simply re-injects the exception back into the VM without any special
14223	* processing.
14224	*
14225	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
14226	*/
14227	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14228	{
14229	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14230
14231	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14232	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14233	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14234	("uVector=%#x u32XcptBitmap=%#X32\n",
14235	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14236	NOREF(pVmcsInfo);
14237	#endif
14238
14239	/*
14240	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14241	* would have been handled while checking exits due to event delivery.
14242	*/
14243	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14244
14245	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14246	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14247	AssertRCReturn(rc, rc);
14248	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14249	#endif
14250
14251	#ifdef VBOX_WITH_STATISTICS
14252	switch (uVector)
14253	{
14254	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14255	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14256	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14257	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14258	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14259	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14260	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14261	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14262	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14263	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14264	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14265	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14266	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14267	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14268	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14269	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14270	default:
14271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14272	break;
14273	}
14274	#endif
14275
14276	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14277	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
14278	NOREF(uVector);
14279
14280	/* Re-inject the original exception into the guest. */
14281	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14282	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14283	return VINF_SUCCESS;
14284	}
14285
14286
14287	/**
14288	* VM-exit exception handler for all exceptions (except NMIs!).
14289	*
14290	* @remarks This may be called for both guests and nested-guests. Take care to not
14291	* make assumptions and avoid doing anything that is not relevant when
14292	* executing a nested-guest (e.g., Mesa driver hacks).
14293	*/
14294	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14295	{
14296	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
14297
14298	/*
14299	* If this VM-exit occurred while delivering an event through the guest IDT, take
14300	* action based on the return code and additional hints (e.g. for page-faults)
14301	* that will be updated in the VMX transient structure.
14302	*/
14303	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14304	if (rcStrict == VINF_SUCCESS)
14305	{
14306	/*
14307	* If an exception caused a VM-exit due to delivery of an event, the original
14308	* event may have to be re-injected into the guest. We shall reinject it and
14309	* continue guest execution. However, page-fault is a complicated case and
14310	* needs additional processing done in hmR0VmxExitXcptPF().
14311	*/
14312	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14313	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14314	if ( !pVCpu->hm.s.Event.fPending
14315	\|\| uVector == X86_XCPT_PF)
14316	{
14317	switch (uVector)
14318	{
14319	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
14320	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
14321	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
14322	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
14323	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
14324	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
14325	default:
14326	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
14327	}
14328	}
14329	/* else: inject pending event before resuming guest execution. */
14330	}
14331	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
14332	{
14333	Assert(pVCpu->hm.s.Event.fPending);
14334	rcStrict = VINF_SUCCESS;
14335	}
14336
14337	return rcStrict;
14338	}
14339	/** @} */
14340
14341
14342	/** @name VM-exit handlers.
14343	* @{
14344	*/
14345	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14346	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14347	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14348
14349	/**
14350	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14351	*/
14352	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14353	{
14354	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14355	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14356	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14357	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14358	return VINF_SUCCESS;
14359	return VINF_EM_RAW_INTERRUPT;
14360	}
14361
14362
14363	/**
14364	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14365	* VM-exit.
14366	*/
14367	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14368	{
14369	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14370	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14371
14372	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14373
14374	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14375	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14376	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14377
14378	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14379	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14380	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14381	NOREF(pVmcsInfo);
14382
14383	VBOXSTRICTRC rcStrict;
14384	switch (uExitIntType)
14385	{
14386	/*
14387	* Host physical NMIs:
14388	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14389	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14390	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14391	*
14392	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14393	* See Intel spec. 27.5.5 "Updating Non-Register State".
14394	*/
14395	case VMX_EXIT_INT_INFO_TYPE_NMI:
14396	{
14397	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14398	break;
14399	}
14400
14401	/*
14402	* Privileged software exceptions (#DB from ICEBP),
14403	* Software exceptions (#BP and #OF),
14404	* Hardware exceptions:
14405	* Process the required exceptions and resume guest execution if possible.
14406	*/
14407	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14408	Assert(uVector == X86_XCPT_DB);
14409	RT_FALL_THRU();
14410	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14411	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14412	RT_FALL_THRU();
14413	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14414	{
14415	NOREF(uVector);
14416	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14417	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14418	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14419	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14420
14421	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14422	break;
14423	}
14424
14425	default:
14426	{
14427	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14428	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14429	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14430	break;
14431	}
14432	}
14433
14434	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14435	return rcStrict;
14436	}
14437
14438
14439	/**
14440	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14441	*/
14442	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14443	{
14444	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14445
14446	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14447	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14448	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14449
14450	/* Evaluate and deliver pending events and resume guest execution. */
14451	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14452	return VINF_SUCCESS;
14453	}
14454
14455
14456	/**
14457	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14458	*/
14459	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14460	{
14461	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14462
14463	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14464	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14465	{
14466	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14467	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14468	}
14469
14470	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14471
14472	/*
14473	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14474	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14475	*/
14476	uint32_t fIntrState;
14477	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14478	AssertRC(rc);
14479	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14480	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14481	{
14482	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14483	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14484
14485	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14486	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14487	AssertRC(rc);
14488	}
14489
14490	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14491	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14492
14493	/* Evaluate and deliver pending events and resume guest execution. */
14494	return VINF_SUCCESS;
14495	}
14496
14497
14498	/**
14499	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14500	*/
14501	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14502	{
14503	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14504	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14505	}
14506
14507
14508	/**
14509	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14510	*/
14511	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14512	{
14513	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14514	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14515	}
14516
14517
14518	/**
14519	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14520	*/
14521	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14522	{
14523	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14524
14525	/*
14526	* Get the state we need and update the exit history entry.
14527	*/
14528	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14529	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14530
14531	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14532	AssertRCReturn(rc, rc);
14533
14534	VBOXSTRICTRC rcStrict;
14535	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14536	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14537	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14538	if (!pExitRec)
14539	{
14540	/*
14541	* Regular CPUID instruction execution.
14542	*/
14543	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
14544	if (rcStrict == VINF_SUCCESS)
14545	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14546	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14547	{
14548	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14549	rcStrict = VINF_SUCCESS;
14550	}
14551	}
14552	else
14553	{
14554	/*
14555	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14556	*/
14557	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14558	AssertRCReturn(rc2, rc2);
14559
14560	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14561	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14562
14563	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14564	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14565
14566	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14567	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14568	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14569	}
14570	return rcStrict;
14571	}
14572
14573
14574	/**
14575	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14576	*/
14577	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14578	{
14579	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14580
14581	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14582	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14583	AssertRCReturn(rc, rc);
14584
14585	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14586	return VINF_EM_RAW_EMULATE_INSTR;
14587
14588	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14589	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14590	}
14591
14592
14593	/**
14594	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14595	*/
14596	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14597	{
14598	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14599
14600	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14601	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14602	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14603	AssertRCReturn(rc, rc);
14604
14605	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
14606	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14607	{
14608	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14609	we must reset offsetting on VM-entry. See @bugref{6634}. */
14610	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14611	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14612	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14613	}
14614	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14615	{
14616	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14617	rcStrict = VINF_SUCCESS;
14618	}
14619	return rcStrict;
14620	}
14621
14622
14623	/**
14624	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14625	*/
14626	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14627	{
14628	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14629
14630	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14631	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14632	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14633	AssertRCReturn(rc, rc);
14634
14635	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
14636	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14637	{
14638	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14639	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14640	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14641	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14642	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14643	}
14644	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14645	{
14646	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14647	rcStrict = VINF_SUCCESS;
14648	}
14649	return rcStrict;
14650	}
14651
14652
14653	/**
14654	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14655	*/
14656	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14657	{
14658	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14659
14660	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14661	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14662	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14663	AssertRCReturn(rc, rc);
14664
14665	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14666	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14667	if (RT_LIKELY(rc == VINF_SUCCESS))
14668	{
14669	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14670	Assert(pVmxTransient->cbExitInstr == 2);
14671	}
14672	else
14673	{
14674	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14675	rc = VERR_EM_INTERPRETER;
14676	}
14677	return rc;
14678	}
14679
14680
14681	/**
14682	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14683	*/
14684	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14685	{
14686	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14687
14688	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14689	if (EMAreHypercallInstructionsEnabled(pVCpu))
14690	{
14691	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14692	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14693	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14694	AssertRCReturn(rc, rc);
14695
14696	/* Perform the hypercall. */
14697	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14698	if (rcStrict == VINF_SUCCESS)
14699	{
14700	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14701	AssertRCReturn(rc, rc);
14702	}
14703	else
14704	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14705	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14706	\|\| RT_FAILURE(rcStrict));
14707
14708	/* If the hypercall changes anything other than guest's general-purpose registers,
14709	we would need to reload the guest changed bits here before VM-entry. */
14710	}
14711	else
14712	Log4Func(("Hypercalls not enabled\n"));
14713
14714	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14715	if (RT_FAILURE(rcStrict))
14716	{
14717	hmR0VmxSetPendingXcptUD(pVCpu);
14718	rcStrict = VINF_SUCCESS;
14719	}
14720
14721	return rcStrict;
14722	}
14723
14724
14725	/**
14726	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14727	*/
14728	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14729	{
14730	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14731	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14732
14733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14734	hmR0VmxReadExitQualVmcs(pVmxTransient);
14735	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14736	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14737	AssertRCReturn(rc, rc);
14738
14739	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
14740
14741	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14742	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14743	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14744	{
14745	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14746	rcStrict = VINF_SUCCESS;
14747	}
14748	else
14749	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14750	VBOXSTRICTRC_VAL(rcStrict)));
14751	return rcStrict;
14752	}
14753
14754
14755	/**
14756	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14757	*/
14758	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14759	{
14760	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14761
14762	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14763	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14764	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14765	AssertRCReturn(rc, rc);
14766
14767	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
14768	if (rcStrict == VINF_SUCCESS)
14769	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14770	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14771	{
14772	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14773	rcStrict = VINF_SUCCESS;
14774	}
14775
14776	return rcStrict;
14777	}
14778
14779
14780	/**
14781	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14782	*/
14783	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14784	{
14785	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14786
14787	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14788	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14789	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14790	AssertRCReturn(rc, rc);
14791
14792	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
14793	if (RT_SUCCESS(rcStrict))
14794	{
14795	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14796	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14797	rcStrict = VINF_SUCCESS;
14798	}
14799
14800	return rcStrict;
14801	}
14802
14803
14804	/**
14805	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14806	* VM-exit.
14807	*/
14808	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14809	{
14810	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14811	return VINF_EM_RESET;
14812	}
14813
14814
14815	/**
14816	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14817	*/
14818	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14819	{
14820	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14821
14822	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14823	AssertRCReturn(rc, rc);
14824
14825	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14826	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14827	rc = VINF_SUCCESS;
14828	else
14829	rc = VINF_EM_HALT;
14830
14831	if (rc != VINF_SUCCESS)
14832	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14833	return rc;
14834	}
14835
14836
14837	/**
14838	* VM-exit handler for instructions that result in a \#UD exception delivered to
14839	* the guest.
14840	*/
14841	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14842	{
14843	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14844	hmR0VmxSetPendingXcptUD(pVCpu);
14845	return VINF_SUCCESS;
14846	}
14847
14848
14849	/**
14850	* VM-exit handler for expiry of the VMX-preemption timer.
14851	*/
14852	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14853	{
14854	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14855
14856	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14857	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14858
14859	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14860	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14861	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14862	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14863	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14864	}
14865
14866
14867	/**
14868	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14869	*/
14870	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14871	{
14872	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14873
14874	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14875	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14876	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14877	AssertRCReturn(rc, rc);
14878
14879	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
14880	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14881	: HM_CHANGED_RAISED_XCPT_MASK);
14882
14883	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14884	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14885
14886	return rcStrict;
14887	}
14888
14889
14890	/**
14891	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14892	*/
14893	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14894	{
14895	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14896
14897	/** @todo Enable the new code after finding a reliably guest test-case. */
14898	#if 1
14899	return VERR_EM_INTERPRETER;
14900	#else
14901	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14902	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14903	hmR0VmxReadExitQualVmcs(pVmxTransient);
14904	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
14905	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14906	AssertRCReturn(rc, rc);
14907
14908	/* Paranoia. Ensure this has a memory operand. */
14909	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
14910
14911	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
14912	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14913	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
14914	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
14915
14916	RTGCPTR GCPtrDesc;
14917	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
14918
14919	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
14920	GCPtrDesc, uType);
14921	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14922	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14923	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14924	{
14925	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14926	rcStrict = VINF_SUCCESS;
14927	}
14928	return rcStrict;
14929	#endif
14930	}
14931
14932
14933	/**
14934	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14935	* VM-exit.
14936	*/
14937	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14938	{
14939	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14940	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14941	AssertRCReturn(rc, rc);
14942
14943	rc = hmR0VmxCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14944	if (RT_FAILURE(rc))
14945	return rc;
14946
14947	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14948	NOREF(uInvalidReason);
14949
14950	#ifdef VBOX_STRICT
14951	uint32_t fIntrState;
14952	uint64_t u64Val;
14953	hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14954	hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14955	hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14956
14957	Log4(("uInvalidReason %u\n", uInvalidReason));
14958	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14959	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14960	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14961
14962	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
14963	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14964	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
14965	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
14966	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
14967	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
14968	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
14969	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14970	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
14971	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
14972	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
14973	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14974	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14975	{
14976	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14977	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14978	}
14979	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
14980	#endif
14981
14982	return VERR_VMX_INVALID_GUEST_STATE;
14983	}
14984
14985	/**
14986	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14987	*/
14988	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14989	{
14990	/*
14991	* Cumulative notes of all recognized but unexpected VM-exits.
14992	*
14993	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
14994	* nested-paging is used.
14995	*
14996	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14997	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14998	* this function (and thereby stop VM execution) for handling such instructions.
14999	*
15000	*
15001	* VMX_EXIT_INIT_SIGNAL:
15002	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
15003	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
15004	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
15005	*
15006	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
15007	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
15008	* See Intel spec. "23.8 Restrictions on VMX operation".
15009	*
15010	* VMX_EXIT_SIPI:
15011	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
15012	* activity state is used. We don't make use of it as our guests don't have direct
15013	* access to the host local APIC.
15014	*
15015	* See Intel spec. 25.3 "Other Causes of VM-exits".
15016	*
15017	* VMX_EXIT_IO_SMI:
15018	* VMX_EXIT_SMI:
15019	* This can only happen if we support dual-monitor treatment of SMI, which can be
15020	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
15021	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
15022	* VMX root mode or receive an SMI. If we get here, something funny is going on.
15023	*
15024	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
15025	* See Intel spec. 25.3 "Other Causes of VM-Exits"
15026	*
15027	* VMX_EXIT_ERR_MSR_LOAD:
15028	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
15029	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
15030	* execution.
15031	*
15032	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
15033	*
15034	* VMX_EXIT_ERR_MACHINE_CHECK:
15035	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
15036	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
15037	* #MC exception abort class exception is raised. We thus cannot assume a
15038	* reasonable chance of continuing any sort of execution and we bail.
15039	*
15040	* See Intel spec. 15.1 "Machine-check Architecture".
15041	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
15042	*
15043	* VMX_EXIT_PML_FULL:
15044	* VMX_EXIT_VIRTUALIZED_EOI:
15045	* VMX_EXIT_APIC_WRITE:
15046	* We do not currently support any of these features and thus they are all unexpected
15047	* VM-exits.
15048	*
15049	* VMX_EXIT_GDTR_IDTR_ACCESS:
15050	* VMX_EXIT_LDTR_TR_ACCESS:
15051	* VMX_EXIT_RDRAND:
15052	* VMX_EXIT_RSM:
15053	* VMX_EXIT_VMFUNC:
15054	* VMX_EXIT_ENCLS:
15055	* VMX_EXIT_RDSEED:
15056	* VMX_EXIT_XSAVES:
15057	* VMX_EXIT_XRSTORS:
15058	* VMX_EXIT_UMWAIT:
15059	* VMX_EXIT_TPAUSE:
15060	* These VM-exits are -not- caused unconditionally by execution of the corresponding
15061	* instruction. Any VM-exit for these instructions indicate a hardware problem,
15062	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
15063	*
15064	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
15065	*/
15066	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15067	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
15068	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15069	}
15070
15071
15072	/**
15073	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
15074	*/
15075	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15076	{
15077	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15078
15079	/** @todo Optimize this: We currently drag in the whole MSR state
15080	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15081	* MSRs required. That would require changes to IEM and possibly CPUM too.
15082	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15083	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15084	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15085	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15086	switch (idMsr)
15087	{
15088	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15089	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15090	}
15091
15092	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15093	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15094	AssertRCReturn(rc, rc);
15095
15096	Log4Func(("ecx=%#RX32\n", idMsr));
15097
15098	#ifdef VBOX_STRICT
15099	Assert(!pVmxTransient->fIsNestedGuest);
15100	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15101	{
15102	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15103	&& idMsr != MSR_K6_EFER)
15104	{
15105	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15106	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15107	}
15108	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15109	{
15110	Assert(pVmcsInfo->pvMsrBitmap);
15111	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15112	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15113	{
15114	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15115	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15116	}
15117	}
15118	}
15119	#endif
15120
15121	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
15122	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15123	if (rcStrict == VINF_SUCCESS)
15124	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15125	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15126	{
15127	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15128	rcStrict = VINF_SUCCESS;
15129	}
15130	else
15131	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
15132	("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15133
15134	return rcStrict;
15135	}
15136
15137
15138	/**
15139	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15140	*/
15141	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15142	{
15143	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15144
15145	/** @todo Optimize this: We currently drag in the whole MSR state
15146	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15147	* MSRs required. That would require changes to IEM and possibly CPUM too.
15148	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15149	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15150	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15151
15152	/*
15153	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15154	* Although we don't need to fetch the base as it will be overwritten shortly, while
15155	* loading guest-state we would also load the entire segment register including limit
15156	* and attributes and thus we need to load them here.
15157	*/
15158	switch (idMsr)
15159	{
15160	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15161	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15162	}
15163
15164	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15165	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15166	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15167	AssertRCReturn(rc, rc);
15168
15169	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15170
15171	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
15172	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15173
15174	if (rcStrict == VINF_SUCCESS)
15175	{
15176	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15177
15178	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15179	if ( idMsr == MSR_IA32_APICBASE
15180	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15181	&& idMsr <= MSR_IA32_X2APIC_END))
15182	{
15183	/*
15184	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15185	* When full APIC register virtualization is implemented we'll have to make
15186	* sure APIC state is saved from the VMCS before IEM changes it.
15187	*/
15188	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15189	}
15190	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15191	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15192	else if (idMsr == MSR_K6_EFER)
15193	{
15194	/*
15195	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15196	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15197	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15198	*/
15199	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15200	}
15201
15202	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15203	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15204	{
15205	switch (idMsr)
15206	{
15207	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15208	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15209	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15210	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15211	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15212	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15213	default:
15214	{
15215	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15216	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15217	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15218	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15219	break;
15220	}
15221	}
15222	}
15223	#ifdef VBOX_STRICT
15224	else
15225	{
15226	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15227	switch (idMsr)
15228	{
15229	case MSR_IA32_SYSENTER_CS:
15230	case MSR_IA32_SYSENTER_EIP:
15231	case MSR_IA32_SYSENTER_ESP:
15232	case MSR_K8_FS_BASE:
15233	case MSR_K8_GS_BASE:
15234	{
15235	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15236	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15237	}
15238
15239	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15240	default:
15241	{
15242	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15243	{
15244	/* EFER MSR writes are always intercepted. */
15245	if (idMsr != MSR_K6_EFER)
15246	{
15247	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15248	idMsr));
15249	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15250	}
15251	}
15252
15253	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15254	{
15255	Assert(pVmcsInfo->pvMsrBitmap);
15256	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15257	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15258	{
15259	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15260	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15261	}
15262	}
15263	break;
15264	}
15265	}
15266	}
15267	#endif /* VBOX_STRICT */
15268	}
15269	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15270	{
15271	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15272	rcStrict = VINF_SUCCESS;
15273	}
15274	else
15275	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
15276	("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15277
15278	return rcStrict;
15279	}
15280
15281
15282	/**
15283	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15284	*/
15285	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15286	{
15287	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15288
15289	/** @todo The guest has likely hit a contended spinlock. We might want to
15290	* poke a schedule different guest VCPU. */
15291	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15292	if (RT_SUCCESS(rc))
15293	return VINF_EM_RAW_INTERRUPT;
15294
15295	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15296	return rc;
15297	}
15298
15299
15300	/**
15301	* VM-exit handler for when the TPR value is lowered below the specified
15302	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15303	*/
15304	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15305	{
15306	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15307	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15308
15309	/*
15310	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15311	* We'll re-evaluate pending interrupts and inject them before the next VM
15312	* entry so we can just continue execution here.
15313	*/
15314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15315	return VINF_SUCCESS;
15316	}
15317
15318
15319	/**
15320	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15321	* VM-exit.
15322	*
15323	* @retval VINF_SUCCESS when guest execution can continue.
15324	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15325	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15326	* incompatible guest state for VMX execution (real-on-v86 case).
15327	*/
15328	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15329	{
15330	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15331	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15332
15333	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15334	hmR0VmxReadExitQualVmcs(pVmxTransient);
15335	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15336
15337	VBOXSTRICTRC rcStrict;
15338	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15339	uint64_t const uExitQual = pVmxTransient->uExitQual;
15340	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15341	switch (uAccessType)
15342	{
15343	/*
15344	* MOV to CRx.
15345	*/
15346	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15347	{
15348	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15349	AssertRCReturn(rc, rc);
15350
15351	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15352	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15353	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15354	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15355
15356	/*
15357	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15358	* - When nested paging isn't used.
15359	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15360	* - We are executing in the VM debug loop.
15361	*/
15362	Assert( iCrReg != 3
15363	\|\| !pVM->hm.s.fNestedPaging
15364	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15365	\|\| pVCpu->hm.s.fUsingDebugLoop);
15366
15367	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15368	Assert( iCrReg != 8
15369	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15370
15371	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15372	AssertMsg( rcStrict == VINF_SUCCESS
15373	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15374
15375	/*
15376	* This is a kludge for handling switches back to real mode when we try to use
15377	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15378	* deal with special selector values, so we have to return to ring-3 and run
15379	* there till the selector values are V86 mode compatible.
15380	*
15381	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15382	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15383	* this function.
15384	*/
15385	if ( iCrReg == 0
15386	&& rcStrict == VINF_SUCCESS
15387	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15388	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15389	&& (uOldCr0 & X86_CR0_PE)
15390	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15391	{
15392	/** @todo Check selectors rather than returning all the time. */
15393	Assert(!pVmxTransient->fIsNestedGuest);
15394	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15395	rcStrict = VINF_EM_RESCHEDULE_REM;
15396	}
15397	break;
15398	}
15399
15400	/*
15401	* MOV from CRx.
15402	*/
15403	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15404	{
15405	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15406	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15407
15408	/*
15409	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15410	* - When nested paging isn't used.
15411	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15412	* - We are executing in the VM debug loop.
15413	*/
15414	Assert( iCrReg != 3
15415	\|\| !pVM->hm.s.fNestedPaging
15416	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15417	\|\| pVCpu->hm.s.fUsingDebugLoop);
15418
15419	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15420	Assert( iCrReg != 8
15421	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15422
15423	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15424	break;
15425	}
15426
15427	/*
15428	* CLTS (Clear Task-Switch Flag in CR0).
15429	*/
15430	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15431	{
15432	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
15433	break;
15434	}
15435
15436	/*
15437	* LMSW (Load Machine-Status Word into CR0).
15438	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15439	*/
15440	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15441	{
15442	RTGCPTR GCPtrEffDst;
15443	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
15444	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15445	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15446	if (fMemOperand)
15447	{
15448	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
15449	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15450	}
15451	else
15452	GCPtrEffDst = NIL_RTGCPTR;
15453	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15454	break;
15455	}
15456
15457	default:
15458	{
15459	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15460	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15461	}
15462	}
15463
15464	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15465	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15466	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15467
15468	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15469	NOREF(pVM);
15470	return rcStrict;
15471	}
15472
15473
15474	/**
15475	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15476	* VM-exit.
15477	*/
15478	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15479	{
15480	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15481	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15482
15483	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15484	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15485	hmR0VmxReadExitQualVmcs(pVmxTransient);
15486	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15487	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15488	\| CPUMCTX_EXTRN_EFER);
15489	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15490	AssertRCReturn(rc, rc);
15491
15492	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15493	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15494	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15495	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15496	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15497	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15498	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15499	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15500
15501	/*
15502	* Update exit history to see if this exit can be optimized.
15503	*/
15504	VBOXSTRICTRC rcStrict;
15505	PCEMEXITREC pExitRec = NULL;
15506	if ( !fGstStepping
15507	&& !fDbgStepping)
15508	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15509	!fIOString
15510	? !fIOWrite
15511	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15512	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15513	: !fIOWrite
15514	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15515	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15516	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15517	if (!pExitRec)
15518	{
15519	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15520	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15521
15522	uint32_t const cbValue = s_aIOSizes[uIOSize];
15523	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
15524	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15525	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15526	if (fIOString)
15527	{
15528	/*
15529	* INS/OUTS - I/O String instruction.
15530	*
15531	* Use instruction-information if available, otherwise fall back on
15532	* interpreting the instruction.
15533	*/
15534	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15535	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15536	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15537	if (fInsOutsInfo)
15538	{
15539	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15540	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15541	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15542	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15543	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15544	if (fIOWrite)
15545	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15546	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15547	else
15548	{
15549	/*
15550	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15551	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15552	* See Intel Instruction spec. for "INS".
15553	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15554	*/
15555	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15556	}
15557	}
15558	else
15559	rcStrict = IEMExecOne(pVCpu);
15560
15561	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15562	fUpdateRipAlready = true;
15563	}
15564	else
15565	{
15566	/*
15567	* IN/OUT - I/O instruction.
15568	*/
15569	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15570	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15571	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15572	if (fIOWrite)
15573	{
15574	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15575	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15576	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15577	&& !pCtx->eflags.Bits.u1TF)
15578	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15579	}
15580	else
15581	{
15582	uint32_t u32Result = 0;
15583	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15584	if (IOM_SUCCESS(rcStrict))
15585	{
15586	/* Save result of I/O IN instr. in AL/AX/EAX. */
15587	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15588	}
15589	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15590	&& !pCtx->eflags.Bits.u1TF)
15591	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15593	}
15594	}
15595
15596	if (IOM_SUCCESS(rcStrict))
15597	{
15598	if (!fUpdateRipAlready)
15599	{
15600	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15601	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15602	}
15603
15604	/*
15605	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15606	* while booting Fedora 17 64-bit guest.
15607	*
15608	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15609	*/
15610	if (fIOString)
15611	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15612
15613	/*
15614	* If any I/O breakpoints are armed, we need to check if one triggered
15615	* and take appropriate action.
15616	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15617	*/
15618	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15619	AssertRCReturn(rc, rc);
15620
15621	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15622	* execution engines about whether hyper BPs and such are pending. */
15623	uint32_t const uDr7 = pCtx->dr[7];
15624	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15625	&& X86_DR7_ANY_RW_IO(uDr7)
15626	&& (pCtx->cr4 & X86_CR4_DE))
15627	\|\| DBGFBpIsHwIoArmed(pVM)))
15628	{
15629	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15630
15631	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15632	VMMRZCallRing3Disable(pVCpu);
15633	HM_DISABLE_PREEMPT(pVCpu);
15634
15635	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15636
15637	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15638	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15639	{
15640	/* Raise #DB. */
15641	if (fIsGuestDbgActive)
15642	ASMSetDR6(pCtx->dr[6]);
15643	if (pCtx->dr[7] != uDr7)
15644	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15645
15646	hmR0VmxSetPendingXcptDB(pVCpu);
15647	}
15648	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15649	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15650	else if ( rcStrict2 != VINF_SUCCESS
15651	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15652	rcStrict = rcStrict2;
15653	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15654
15655	HM_RESTORE_PREEMPT();
15656	VMMRZCallRing3Enable(pVCpu);
15657	}
15658	}
15659
15660	#ifdef VBOX_STRICT
15661	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15662	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15663	Assert(!fIOWrite);
15664	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15665	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15666	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15667	Assert(fIOWrite);
15668	else
15669	{
15670	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15671	* statuses, that the VMM device and some others may return. See
15672	* IOM_SUCCESS() for guidance. */
15673	AssertMsg( RT_FAILURE(rcStrict)
15674	\|\| rcStrict == VINF_SUCCESS
15675	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15676	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15677	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15678	\|\| rcStrict == VINF_EM_RAW_TO_R3
15679	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15680	# endif
15681	}
15682	#endif
15683	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15684	}
15685	else
15686	{
15687	/*
15688	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15689	*/
15690	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15691	AssertRCReturn(rc2, rc2);
15692	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15693	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15694	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15695	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15696	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15697	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15698
15699	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15700	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15701
15702	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15703	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15704	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15705	}
15706	return rcStrict;
15707	}
15708
15709
15710	/**
15711	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15712	* VM-exit.
15713	*/
15714	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15715	{
15716	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15717
15718	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15719	hmR0VmxReadExitQualVmcs(pVmxTransient);
15720	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15721	{
15722	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15723	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15724	{
15725	uint32_t uErrCode;
15726	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15727	{
15728	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15729	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15730	}
15731	else
15732	uErrCode = 0;
15733
15734	RTGCUINTPTR GCPtrFaultAddress;
15735	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15736	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15737	else
15738	GCPtrFaultAddress = 0;
15739
15740	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15741
15742	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15743	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
15744
15745	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15746	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15747	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15748	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15749	}
15750	}
15751
15752	/* Fall back to the interpreter to emulate the task-switch. */
15753	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15754	return VERR_EM_INTERPRETER;
15755	}
15756
15757
15758	/**
15759	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15760	*/
15761	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15762	{
15763	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15764
15765	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15766	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15767	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15768	AssertRC(rc);
15769	return VINF_EM_DBG_STEPPED;
15770	}
15771
15772
15773	/**
15774	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15775	*/
15776	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15777	{
15778	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15779	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15780
15781	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15782	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15783	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15784	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15785	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15786
15787	/*
15788	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15789	*/
15790	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15791	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15792	{
15793	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15794	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15795	{
15796	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15797	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15798	}
15799	}
15800	else
15801	{
15802	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15803	return rcStrict;
15804	}
15805
15806	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15807	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15808	hmR0VmxReadExitQualVmcs(pVmxTransient);
15809	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15810	AssertRCReturn(rc, rc);
15811
15812	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15813	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15814	switch (uAccessType)
15815	{
15816	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15817	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15818	{
15819	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15820	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15821	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15822
15823	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15824	GCPhys &= PAGE_BASE_GC_MASK;
15825	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15826	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15827	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15828
15829	rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
15830	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
15831	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15832	if ( rcStrict == VINF_SUCCESS
15833	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15834	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15835	{
15836	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15837	\| HM_CHANGED_GUEST_APIC_TPR);
15838	rcStrict = VINF_SUCCESS;
15839	}
15840	break;
15841	}
15842
15843	default:
15844	{
15845	Log4Func(("uAccessType=%#x\n", uAccessType));
15846	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15847	break;
15848	}
15849	}
15850
15851	if (rcStrict != VINF_SUCCESS)
15852	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15853	return rcStrict;
15854	}
15855
15856
15857	/**
15858	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15859	* VM-exit.
15860	*/
15861	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15862	{
15863	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15864	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15865
15866	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15867	if (!pVmxTransient->fIsNestedGuest)
15868	{
15869	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15870	if (pVmxTransient->fWasGuestDebugStateActive)
15871	{
15872	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15873	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15874	}
15875
15876	if ( !pVCpu->hm.s.fSingleInstruction
15877	&& !pVmxTransient->fWasHyperDebugStateActive)
15878	{
15879	Assert(!DBGFIsStepping(pVCpu));
15880	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15881
15882	/* Don't intercept MOV DRx any more. */
15883	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15884	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15885	AssertRC(rc);
15886
15887	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15888	VMMRZCallRing3Disable(pVCpu);
15889	HM_DISABLE_PREEMPT(pVCpu);
15890
15891	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15892	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15893	Assert(CPUMIsGuestDebugStateActive(pVCpu));
15894
15895	HM_RESTORE_PREEMPT();
15896	VMMRZCallRing3Enable(pVCpu);
15897
15898	#ifdef VBOX_WITH_STATISTICS
15899	hmR0VmxReadExitQualVmcs(pVmxTransient);
15900	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15901	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15902	else
15903	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15904	#endif
15905	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15906	return VINF_SUCCESS;
15907	}
15908	}
15909
15910	/*
15911	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15912	* The EFER MSR is always up-to-date.
15913	* Update the segment registers and DR7 from the CPU.
15914	*/
15915	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15916	hmR0VmxReadExitQualVmcs(pVmxTransient);
15917	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15918	AssertRCReturn(rc, rc);
15919	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15920
15921	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15922	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15923	{
15924	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15925	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15926	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15927	if (RT_SUCCESS(rc))
15928	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15930	}
15931	else
15932	{
15933	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15934	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15935	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15936	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15937	}
15938
15939	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15940	if (RT_SUCCESS(rc))
15941	{
15942	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15943	AssertRCReturn(rc2, rc2);
15944	return VINF_SUCCESS;
15945	}
15946	return rc;
15947	}
15948
15949
15950	/**
15951	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15952	* Conditional VM-exit.
15953	*/
15954	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15955	{
15956	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15957	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15958
15959	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15960	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15961	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15962	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15963	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15964
15965	/*
15966	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15967	*/
15968	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15969	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15970	{
15971	/*
15972	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15973	* instruction emulation to inject the original event. Otherwise, injecting the original event
15974	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
15975	*/
15976	if (!pVCpu->hm.s.Event.fPending)
15977	{ /* likely */ }
15978	else
15979	{
15980	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15981	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15982	/** @todo NSTVMX: Think about how this should be handled. */
15983	if (pVmxTransient->fIsNestedGuest)
15984	return VERR_VMX_IPE_3;
15985	#endif
15986	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15987	}
15988	}
15989	else
15990	{
15991	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15992	return rcStrict;
15993	}
15994
15995	/*
15996	* Get sufficient state and update the exit history entry.
15997	*/
15998	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15999	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
16000	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16001	AssertRCReturn(rc, rc);
16002
16003	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
16004	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
16005	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
16006	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
16007	if (!pExitRec)
16008	{
16009	/*
16010	* If we succeed, resume guest execution.
16011	* If we fail in interpreting the instruction because we couldn't get the guest physical address
16012	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
16013	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
16014	* weird case. See @bugref{6043}.
16015	*/
16016	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16017	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16018	/** @todo bird: We can probably just go straight to IOM here and assume that
16019	* it's MMIO, then fall back on PGM if that hunch didn't work out so
16020	* well. However, we need to address that aliasing workarounds that
16021	* PGMR0Trap0eHandlerNPMisconfig implements. So, some care is needed.
16022	*
16023	* Might also be interesting to see if we can get this done more or
16024	* less locklessly inside IOM. Need to consider the lookup table
16025	* updating and use a bit more carefully first (or do all updates via
16026	* rendezvous) */
16027	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
16028	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
16029	if ( rcStrict == VINF_SUCCESS
16030	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16031	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16032	{
16033	/* Successfully handled MMIO operation. */
16034	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16035	\| HM_CHANGED_GUEST_APIC_TPR);
16036	rcStrict = VINF_SUCCESS;
16037	}
16038	}
16039	else
16040	{
16041	/*
16042	* Frequent exit or something needing probing. Call EMHistoryExec.
16043	*/
16044	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
16045	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
16046
16047	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16048	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16049
16050	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16051	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16052	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16053	}
16054	return rcStrict;
16055	}
16056
16057
16058	/**
16059	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
16060	* VM-exit.
16061	*/
16062	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16063	{
16064	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16065	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16066
16067	hmR0VmxReadExitQualVmcs(pVmxTransient);
16068	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16069	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16070	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16071	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16072	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16073
16074	/*
16075	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
16076	*/
16077	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16078	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16079	{
16080	/*
16081	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
16082	* we shall resolve the nested #PF and re-inject the original event.
16083	*/
16084	if (pVCpu->hm.s.Event.fPending)
16085	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
16086	}
16087	else
16088	{
16089	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
16090	return rcStrict;
16091	}
16092
16093	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16094	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
16095	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16096	AssertRCReturn(rc, rc);
16097
16098	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
16099	uint64_t const uExitQual = pVmxTransient->uExitQual;
16100	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
16101
16102	RTGCUINT uErrorCode = 0;
16103	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16104	uErrorCode \|= X86_TRAP_PF_ID;
16105	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16106	uErrorCode \|= X86_TRAP_PF_RW;
16107	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16108	uErrorCode \|= X86_TRAP_PF_P;
16109
16110	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16111	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16112	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
16113
16114	/*
16115	* Handle the pagefault trap for the nested shadow table.
16116	*/
16117	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16118	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16119	TRPMResetTrap(pVCpu);
16120
16121	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16122	if ( rcStrict == VINF_SUCCESS
16123	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16124	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16125	{
16126	/* Successfully synced our nested page tables. */
16127	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16128	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16129	return VINF_SUCCESS;
16130	}
16131
16132	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16133	return rcStrict;
16134	}
16135
16136
16137	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16138	/**
16139	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16140	*/
16141	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16142	{
16143	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16144
16145	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16146	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16147	hmR0VmxReadExitQualVmcs(pVmxTransient);
16148	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16149	\| CPUMCTX_EXTRN_HWVIRT
16150	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16151	AssertRCReturn(rc, rc);
16152
16153	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16154
16155	VMXVEXITINFO ExitInfo;
16156	RT_ZERO(ExitInfo);
16157	ExitInfo.uReason = pVmxTransient->uExitReason;
16158	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16159	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16160	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16161	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16162
16163	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16164	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16165	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16166	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16167	{
16168	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16169	rcStrict = VINF_SUCCESS;
16170	}
16171	return rcStrict;
16172	}
16173
16174
16175	/**
16176	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16177	*/
16178	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16179	{
16180	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16181
16182	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16183	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16184	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16185	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16186	AssertRCReturn(rc, rc);
16187
16188	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16189
16190	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16191	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
16192	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16193	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16194	{
16195	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16196	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16197	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16198	}
16199	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16200	return rcStrict;
16201	}
16202
16203
16204	/**
16205	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16206	*/
16207	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16208	{
16209	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16210
16211	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16212	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16213	hmR0VmxReadExitQualVmcs(pVmxTransient);
16214	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16215	\| CPUMCTX_EXTRN_HWVIRT
16216	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16217	AssertRCReturn(rc, rc);
16218
16219	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16220
16221	VMXVEXITINFO ExitInfo;
16222	RT_ZERO(ExitInfo);
16223	ExitInfo.uReason = pVmxTransient->uExitReason;
16224	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16225	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16226	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16227	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16228
16229	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16230	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16231	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16232	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16233	{
16234	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16235	rcStrict = VINF_SUCCESS;
16236	}
16237	return rcStrict;
16238	}
16239
16240
16241	/**
16242	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16243	*/
16244	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16245	{
16246	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16247
16248	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16249	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16250	hmR0VmxReadExitQualVmcs(pVmxTransient);
16251	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16252	\| CPUMCTX_EXTRN_HWVIRT
16253	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16254	AssertRCReturn(rc, rc);
16255
16256	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16257
16258	VMXVEXITINFO ExitInfo;
16259	RT_ZERO(ExitInfo);
16260	ExitInfo.uReason = pVmxTransient->uExitReason;
16261	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16262	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16263	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16264	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16265
16266	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16267	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16268	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16269	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16270	{
16271	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16272	rcStrict = VINF_SUCCESS;
16273	}
16274	return rcStrict;
16275	}
16276
16277
16278	/**
16279	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16280	*/
16281	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16282	{
16283	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16284
16285	/*
16286	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16287	* thus might not need to import the shadow VMCS state, it's safer just in case
16288	* code elsewhere dares look at unsynced VMCS fields.
16289	*/
16290	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16291	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16292	hmR0VmxReadExitQualVmcs(pVmxTransient);
16293	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16294	\| CPUMCTX_EXTRN_HWVIRT
16295	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16296	AssertRCReturn(rc, rc);
16297
16298	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16299
16300	VMXVEXITINFO ExitInfo;
16301	RT_ZERO(ExitInfo);
16302	ExitInfo.uReason = pVmxTransient->uExitReason;
16303	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16304	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16305	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16306	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16307	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16308
16309	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16310	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16311	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16312	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16313	{
16314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16315	rcStrict = VINF_SUCCESS;
16316	}
16317	return rcStrict;
16318	}
16319
16320
16321	/**
16322	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16323	*/
16324	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16325	{
16326	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16327
16328	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16329	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16330	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16331	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16332	AssertRCReturn(rc, rc);
16333
16334	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16335
16336	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16337	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
16338	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16339	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16340	{
16341	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16342	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16343	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16344	}
16345	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16346	return rcStrict;
16347	}
16348
16349
16350	/**
16351	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16352	*/
16353	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16354	{
16355	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16356
16357	/*
16358	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16359	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16360	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16361	*/
16362	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16363	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16364	hmR0VmxReadExitQualVmcs(pVmxTransient);
16365	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16366	\| CPUMCTX_EXTRN_HWVIRT
16367	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16368	AssertRCReturn(rc, rc);
16369
16370	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16371
16372	VMXVEXITINFO ExitInfo;
16373	RT_ZERO(ExitInfo);
16374	ExitInfo.uReason = pVmxTransient->uExitReason;
16375	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16376	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16377	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16378	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16379	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16380
16381	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16382	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16383	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16384	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16385	{
16386	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16387	rcStrict = VINF_SUCCESS;
16388	}
16389	return rcStrict;
16390	}
16391
16392
16393	/**
16394	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16395	*/
16396	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16397	{
16398	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16399
16400	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16401	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16402	\| CPUMCTX_EXTRN_HWVIRT
16403	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16404	AssertRCReturn(rc, rc);
16405
16406	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16407
16408	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
16409	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16410	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16411	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16412	{
16413	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16414	rcStrict = VINF_SUCCESS;
16415	}
16416	return rcStrict;
16417	}
16418
16419
16420	/**
16421	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16422	*/
16423	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16424	{
16425	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16426
16427	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16428	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16429	hmR0VmxReadExitQualVmcs(pVmxTransient);
16430	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16431	\| CPUMCTX_EXTRN_HWVIRT
16432	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16433	AssertRCReturn(rc, rc);
16434
16435	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16436
16437	VMXVEXITINFO ExitInfo;
16438	RT_ZERO(ExitInfo);
16439	ExitInfo.uReason = pVmxTransient->uExitReason;
16440	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16441	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16442	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16443	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16444
16445	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16446	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16447	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16448	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16449	{
16450	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16451	rcStrict = VINF_SUCCESS;
16452	}
16453	return rcStrict;
16454	}
16455
16456
16457	/**
16458	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16459	*/
16460	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16461	{
16462	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16463
16464	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16465	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16466	hmR0VmxReadExitQualVmcs(pVmxTransient);
16467	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16468	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16469	AssertRCReturn(rc, rc);
16470
16471	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16472
16473	VMXVEXITINFO ExitInfo;
16474	RT_ZERO(ExitInfo);
16475	ExitInfo.uReason = pVmxTransient->uExitReason;
16476	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16477	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16478	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16479	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16480
16481	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16482	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16483	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16484	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16485	{
16486	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16487	rcStrict = VINF_SUCCESS;
16488	}
16489	return rcStrict;
16490	}
16491	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16492	/** @} */
16493
16494
16495	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16496	/** @name Nested-guest VM-exit handlers.
16497	* @{
16498	*/
16499	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16500	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16501	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16502
16503	/**
16504	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16505	* Conditional VM-exit.
16506	*/
16507	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16508	{
16509	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16510
16511	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16512
16513	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16514	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16515	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16516
16517	switch (uExitIntType)
16518	{
16519	/*
16520	* Physical NMIs:
16521	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16522	*/
16523	case VMX_EXIT_INT_INFO_TYPE_NMI:
16524	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16525
16526	/*
16527	* Hardware exceptions,
16528	* Software exceptions,
16529	* Privileged software exceptions:
16530	* Figure out if the exception must be delivered to the guest or the nested-guest.
16531	*/
16532	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16533	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16534	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16535	{
16536	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16537	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16538	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16539	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16540
16541	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16542	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16543	pVmxTransient->uExitIntErrorCode);
16544	if (fIntercept)
16545	{
16546	/* Exit qualification is required for debug and page-fault exceptions. */
16547	hmR0VmxReadExitQualVmcs(pVmxTransient);
16548
16549	/*
16550	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16551	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16552	* length. However, if delivery of a software interrupt, software exception or privileged
16553	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16554	*/
16555	VMXVEXITINFO ExitInfo;
16556	RT_ZERO(ExitInfo);
16557	ExitInfo.uReason = pVmxTransient->uExitReason;
16558	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16559	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16560
16561	VMXVEXITEVENTINFO ExitEventInfo;
16562	RT_ZERO(ExitEventInfo);
16563	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16564	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16565	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16566	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16567
16568	#ifdef DEBUG_ramshankar
16569	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16570	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
16571	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16572	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16573	{
16574	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
16575	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
16576	}
16577	#endif
16578	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16579	}
16580
16581	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16582	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16583	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16584	}
16585
16586	/*
16587	* Software interrupts:
16588	* VM-exits cannot be caused by software interrupts.
16589	*
16590	* External interrupts:
16591	* This should only happen when "acknowledge external interrupts on VM-exit"
16592	* control is set. However, we never set this when executing a guest or
16593	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16594	* the guest.
16595	*/
16596	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16597	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16598	default:
16599	{
16600	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16601	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16602	}
16603	}
16604	}
16605
16606
16607	/**
16608	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16609	* Unconditional VM-exit.
16610	*/
16611	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16612	{
16613	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16614	return IEMExecVmxVmexitTripleFault(pVCpu);
16615	}
16616
16617
16618	/**
16619	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16620	*/
16621	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16622	{
16623	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16624
16625	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16626	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16627	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16628	}
16629
16630
16631	/**
16632	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16633	*/
16634	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16635	{
16636	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16637
16638	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16639	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16640	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16641	}
16642
16643
16644	/**
16645	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16646	* Unconditional VM-exit.
16647	*/
16648	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16649	{
16650	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16651
16652	hmR0VmxReadExitQualVmcs(pVmxTransient);
16653	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16654	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16655	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16656
16657	VMXVEXITINFO ExitInfo;
16658	RT_ZERO(ExitInfo);
16659	ExitInfo.uReason = pVmxTransient->uExitReason;
16660	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16661	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16662
16663	VMXVEXITEVENTINFO ExitEventInfo;
16664	RT_ZERO(ExitEventInfo);
16665	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16666	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16667	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16668	}
16669
16670
16671	/**
16672	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16673	*/
16674	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16675	{
16676	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16677
16678	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16679	{
16680	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16681	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16682	}
16683	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16684	}
16685
16686
16687	/**
16688	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16689	*/
16690	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16691	{
16692	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16693
16694	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16695	{
16696	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16697	hmR0VmxReadExitQualVmcs(pVmxTransient);
16698
16699	VMXVEXITINFO ExitInfo;
16700	RT_ZERO(ExitInfo);
16701	ExitInfo.uReason = pVmxTransient->uExitReason;
16702	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16703	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16704	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16705	}
16706	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16707	}
16708
16709
16710	/**
16711	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16712	*/
16713	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16714	{
16715	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16716
16717	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16718	{
16719	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16720	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16721	}
16722	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16723	}
16724
16725
16726	/**
16727	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16728	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16729	*/
16730	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16731	{
16732	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16733
16734	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16735	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16736
16737	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16738
16739	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16740	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16741	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16742
16743	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16744	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16745	u64VmcsField &= UINT64_C(0xffffffff);
16746
16747	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16748	{
16749	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16750	hmR0VmxReadExitQualVmcs(pVmxTransient);
16751
16752	VMXVEXITINFO ExitInfo;
16753	RT_ZERO(ExitInfo);
16754	ExitInfo.uReason = pVmxTransient->uExitReason;
16755	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16756	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16757	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16758	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16759	}
16760
16761	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16762	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16763	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16764	}
16765
16766
16767	/**
16768	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16769	*/
16770	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16771	{
16772	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16773
16774	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16775	{
16776	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16777	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16778	}
16779
16780	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16781	}
16782
16783
16784	/**
16785	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16786	* Conditional VM-exit.
16787	*/
16788	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16789	{
16790	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16791
16792	hmR0VmxReadExitQualVmcs(pVmxTransient);
16793	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16794
16795	VBOXSTRICTRC rcStrict;
16796	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16797	switch (uAccessType)
16798	{
16799	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16800	{
16801	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16802	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16803	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16804	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16805
16806	bool fIntercept;
16807	switch (iCrReg)
16808	{
16809	case 0:
16810	case 4:
16811	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16812	break;
16813
16814	case 3:
16815	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16816	break;
16817
16818	case 8:
16819	fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16820	break;
16821
16822	default:
16823	fIntercept = false;
16824	break;
16825	}
16826	if (fIntercept)
16827	{
16828	VMXVEXITINFO ExitInfo;
16829	RT_ZERO(ExitInfo);
16830	ExitInfo.uReason = pVmxTransient->uExitReason;
16831	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16832	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16833	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16834	}
16835	else
16836	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16837	break;
16838	}
16839
16840	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16841	{
16842	/*
16843	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16844	* CR2 reads do not cause a VM-exit.
16845	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16846	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16847	*/
16848	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16849	if ( iCrReg == 3
16850	\|\| iCrReg == 8)
16851	{
16852	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16853	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16854	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16855	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
16856	{
16857	VMXVEXITINFO ExitInfo;
16858	RT_ZERO(ExitInfo);
16859	ExitInfo.uReason = pVmxTransient->uExitReason;
16860	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16861	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16862	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16863	}
16864	else
16865	{
16866	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16867	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16868	}
16869	}
16870	else
16871	{
16872	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16873	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16874	}
16875	break;
16876	}
16877
16878	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16879	{
16880	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16881	Assert(pVmcsNstGst);
16882	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16883	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16884	if ( (uGstHostMask & X86_CR0_TS)
16885	&& (uReadShadow & X86_CR0_TS))
16886	{
16887	VMXVEXITINFO ExitInfo;
16888	RT_ZERO(ExitInfo);
16889	ExitInfo.uReason = pVmxTransient->uExitReason;
16890	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16891	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16892	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16893	}
16894	else
16895	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
16896	break;
16897	}
16898
16899	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16900	{
16901	RTGCPTR GCPtrEffDst;
16902	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16903	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16904	if (fMemOperand)
16905	{
16906	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16907	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16908	}
16909	else
16910	GCPtrEffDst = NIL_RTGCPTR;
16911
16912	if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
16913	{
16914	VMXVEXITINFO ExitInfo;
16915	RT_ZERO(ExitInfo);
16916	ExitInfo.uReason = pVmxTransient->uExitReason;
16917	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16918	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16919	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16920	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16921	}
16922	else
16923	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
16924	break;
16925	}
16926
16927	default:
16928	{
16929	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16930	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16931	}
16932	}
16933
16934	if (rcStrict == VINF_IEM_RAISED_XCPT)
16935	{
16936	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16937	rcStrict = VINF_SUCCESS;
16938	}
16939	return rcStrict;
16940	}
16941
16942
16943	/**
16944	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16945	* Conditional VM-exit.
16946	*/
16947	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16948	{
16949	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16950
16951	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16952	{
16953	hmR0VmxReadExitQualVmcs(pVmxTransient);
16954	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16955
16956	VMXVEXITINFO ExitInfo;
16957	RT_ZERO(ExitInfo);
16958	ExitInfo.uReason = pVmxTransient->uExitReason;
16959	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16960	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16961	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16962	}
16963	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16964	}
16965
16966
16967	/**
16968	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16969	* Conditional VM-exit.
16970	*/
16971	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16972	{
16973	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16974
16975	hmR0VmxReadExitQualVmcs(pVmxTransient);
16976
16977	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16978	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16979	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16980
16981	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16982	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16983	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16984	{
16985	/*
16986	* IN/OUT instruction:
16987	* - Provides VM-exit instruction length.
16988	*
16989	* INS/OUTS instruction:
16990	* - Provides VM-exit instruction length.
16991	* - Provides Guest-linear address.
16992	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16993	*/
16994	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16995	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16996
16997	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16998	pVmxTransient->ExitInstrInfo.u = 0;
16999	pVmxTransient->uGuestLinearAddr = 0;
17000
17001	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
17002	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
17003	if (fIOString)
17004	{
17005	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
17006	if (fVmxInsOutsInfo)
17007	{
17008	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
17009	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17010	}
17011	}
17012
17013	VMXVEXITINFO ExitInfo;
17014	RT_ZERO(ExitInfo);
17015	ExitInfo.uReason = pVmxTransient->uExitReason;
17016	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17017	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17018	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17019	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
17020	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17021	}
17022	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
17023	}
17024
17025
17026	/**
17027	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
17028	*/
17029	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17030	{
17031	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17032
17033	uint32_t fMsrpm;
17034	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17035	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17036	else
17037	fMsrpm = VMXMSRPM_EXIT_RD;
17038
17039	if (fMsrpm & VMXMSRPM_EXIT_RD)
17040	{
17041	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17042	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17043	}
17044	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
17045	}
17046
17047
17048	/**
17049	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
17050	*/
17051	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17052	{
17053	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17054
17055	uint32_t fMsrpm;
17056	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17057	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17058	else
17059	fMsrpm = VMXMSRPM_EXIT_WR;
17060
17061	if (fMsrpm & VMXMSRPM_EXIT_WR)
17062	{
17063	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17064	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17065	}
17066	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17067	}
17068
17069
17070	/**
17071	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17072	*/
17073	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17074	{
17075	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17076
17077	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17078	{
17079	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17080	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17081	}
17082	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17083	}
17084
17085
17086	/**
17087	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17088	* VM-exit.
17089	*/
17090	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17091	{
17092	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17093
17094	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17095	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
17096	VMXVEXITINFO ExitInfo;
17097	RT_ZERO(ExitInfo);
17098	ExitInfo.uReason = pVmxTransient->uExitReason;
17099	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
17100	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
17101	}
17102
17103
17104	/**
17105	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17106	*/
17107	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17108	{
17109	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17110
17111	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17112	{
17113	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17114	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17115	}
17116	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17117	}
17118
17119
17120	/**
17121	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17122	*/
17123	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17124	{
17125	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17126
17127	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17128	* PAUSE when executing a nested-guest? If it does not, we would not need
17129	* to check for the intercepts here. Just call VM-exit... */
17130
17131	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17132	if ( CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17133	\|\| CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17134	{
17135	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17136	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17137	}
17138	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17139	}
17140
17141
17142	/**
17143	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17144	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17145	*/
17146	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17147	{
17148	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17149
17150	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17151	{
17152	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
17153	VMXVEXITINFO ExitInfo;
17154	RT_ZERO(ExitInfo);
17155	ExitInfo.uReason = pVmxTransient->uExitReason;
17156	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
17157	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
17158	}
17159	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17160	}
17161
17162
17163	/**
17164	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17165	* VM-exit.
17166	*/
17167	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17168	{
17169	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17170
17171	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17172	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17173	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17174	hmR0VmxReadExitQualVmcs(pVmxTransient);
17175
17176	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17177
17178	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
17179	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
17180
17181	VMXVEXITINFO ExitInfo;
17182	RT_ZERO(ExitInfo);
17183	ExitInfo.uReason = pVmxTransient->uExitReason;
17184	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17185	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17186
17187	VMXVEXITEVENTINFO ExitEventInfo;
17188	RT_ZERO(ExitEventInfo);
17189	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17190	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17191	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17192	}
17193
17194
17195	/**
17196	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17197	* Conditional VM-exit.
17198	*/
17199	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17200	{
17201	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17202
17203	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17204	hmR0VmxReadExitQualVmcs(pVmxTransient);
17205	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17206	}
17207
17208
17209	/**
17210	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17211	* Conditional VM-exit.
17212	*/
17213	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17214	{
17215	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17216
17217	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17218	hmR0VmxReadExitQualVmcs(pVmxTransient);
17219	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17220	}
17221
17222
17223	/**
17224	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17225	*/
17226	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17227	{
17228	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17229
17230	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17231	{
17232	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17233	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17234	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17235	}
17236	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17237	}
17238
17239
17240	/**
17241	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17242	*/
17243	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17244	{
17245	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17246
17247	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17248	{
17249	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17250	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17251	}
17252	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17253	}
17254
17255
17256	/**
17257	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17258	*/
17259	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17260	{
17261	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17262
17263	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17264	{
17265	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17266	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17267	hmR0VmxReadExitQualVmcs(pVmxTransient);
17268	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17269
17270	VMXVEXITINFO ExitInfo;
17271	RT_ZERO(ExitInfo);
17272	ExitInfo.uReason = pVmxTransient->uExitReason;
17273	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17274	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17275	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17276	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17277	}
17278	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17279	}
17280
17281
17282	/**
17283	* Nested-guest VM-exit handler for invalid-guest state
17284	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17285	*/
17286	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17287	{
17288	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17289
17290	/*
17291	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17292	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17293	* Handle it like it's in an invalid guest state of the outer guest.
17294	*
17295	* When the fast path is implemented, this should be changed to cause the corresponding
17296	* nested-guest VM-exit.
17297	*/
17298	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17299	}
17300
17301
17302	/**
17303	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17304	* and only provide the instruction length.
17305	*
17306	* Unconditional VM-exit.
17307	*/
17308	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17309	{
17310	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17311
17312	#ifdef VBOX_STRICT
17313	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17314	switch (pVmxTransient->uExitReason)
17315	{
17316	case VMX_EXIT_ENCLS:
17317	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17318	break;
17319
17320	case VMX_EXIT_VMFUNC:
17321	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
17322	break;
17323	}
17324	#endif
17325
17326	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17327	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17328	}
17329
17330
17331	/**
17332	* Nested-guest VM-exit handler for instructions that provide instruction length as
17333	* well as more information.
17334	*
17335	* Unconditional VM-exit.
17336	*/
17337	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17338	{
17339	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17340
17341	#ifdef VBOX_STRICT
17342	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17343	switch (pVmxTransient->uExitReason)
17344	{
17345	case VMX_EXIT_GDTR_IDTR_ACCESS:
17346	case VMX_EXIT_LDTR_TR_ACCESS:
17347	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17348	break;
17349
17350	case VMX_EXIT_RDRAND:
17351	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17352	break;
17353
17354	case VMX_EXIT_RDSEED:
17355	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17356	break;
17357
17358	case VMX_EXIT_XSAVES:
17359	case VMX_EXIT_XRSTORS:
17360	/** @todo NSTVMX: Verify XSS-bitmap. */
17361	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17362	break;
17363
17364	case VMX_EXIT_UMWAIT:
17365	case VMX_EXIT_TPAUSE:
17366	Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17367	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17368	break;
17369	}
17370	#endif
17371
17372	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17373	hmR0VmxReadExitQualVmcs(pVmxTransient);
17374	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17375
17376	VMXVEXITINFO ExitInfo;
17377	RT_ZERO(ExitInfo);
17378	ExitInfo.uReason = pVmxTransient->uExitReason;
17379	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17380	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17381	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17382	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17383	}
17384
17385	/** @} */
17386	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17387

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 87390

Download in other formats: