HMVMXR0.cpp@ 72992

Last change on this file since 72992 was 72992, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Stat nits.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 563.1 KB

Line
1	/* $Id: HMVMXR0.cpp 72992 2018-07-09 03:56:00Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/gim.h>
35	#include <VBox/vmm/apic.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#include "HMInternal.h"
40	#include <VBox/vmm/vm.h>
41	#include "HMVMXR0.h"
42	#include "dtrace/VBoxVMM.h"
43
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50	# define HMVMX_ALWAYS_TRAP_PF
51	# define HMVMX_ALWAYS_FLUSH_TLB
52	# define HMVMX_ALWAYS_SWAP_EFER
53	#endif
54
55
56	/*********************************************************************************************************************************
57	* Defined Constants And Macros *
58	*********************************************************************************************************************************/
59	/** Use the function table. */
60	#define HMVMX_USE_FUNCTION_TABLE
61
62	/** Determine which tagged-TLB flush handler to use. */
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68	/** @name HMVMX_READ_XXX
69	* Flags to skip redundant reads of some common VMCS fields that are not part of
70	* the guest-CPU or VCPU state but are needed while handling VM-exits.
71	*/
72	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79	/** @} */
80
81	/**
82	* States of the VMCS.
83	*
84	* This does not reflect all possible VMCS states but currently only those
85	* needed for maintaining the VMCS consistently even when thread-context hooks
86	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
87	*/
88	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92	/**
93	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94	* guest using hardware-assisted VMX.
95	*
96	* This excludes state like GPRs (other than RSP) which are always are
97	* swapped and restored across the world-switch and also registers like EFER,
98	* MSR which cannot be modified by the guest without causing a VM-exit.
99	*/
100	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101	\| CPUMCTX_EXTRN_RFLAGS \
102	\| CPUMCTX_EXTRN_RSP \
103	\| CPUMCTX_EXTRN_SREG_MASK \
104	\| CPUMCTX_EXTRN_TABLE_MASK \
105	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
106	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
107	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
108	\| CPUMCTX_EXTRN_TSC_AUX \
109	\| CPUMCTX_EXTRN_OTHER_MSRS \
110	\| CPUMCTX_EXTRN_CR0 \
111	\| CPUMCTX_EXTRN_CR3 \
112	\| CPUMCTX_EXTRN_CR4 \
113	\| CPUMCTX_EXTRN_DR7 \
114	\| CPUMCTX_EXTRN_HM_VMX_MASK)
115
116	/**
117	* Exception bitmap mask for real-mode guests (real-on-v86).
118	*
119	* We need to intercept all exceptions manually except:
120	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121	* due to bugs in Intel CPUs.
122	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
123	* support.
124	*/
125	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
126	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
127	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
128	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
129	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
131	\| RT_BIT(X86_XCPT_XF))
132
133	/** Maximum VM-instruction error number. */
134	#define HMVMX_INSTR_ERROR_MAX 28
135
136	/** Profiling macro. */
137	#ifdef HM_PROFILE_EXIT_DISPATCH
138	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140	#else
141	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143	#endif
144
145	/** Assert that preemption is disabled or covered by thread-context hooks. */
146	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
147	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149	/** Assert that we haven't migrated CPUs when thread-context hooks are not
150	* used. */
151	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
152	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
153	("Illegal migration! Entered on CPU %u Current %u\n", \
154	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
155
156	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157	* context. */
158	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162	/** Helper macro for VM-exit handlers called unexpectedly. */
163	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164	do { \
165	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166	return VERR_VMX_UNEXPECTED_EXIT; \
167	} while (0)
168
169	/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171	# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172	hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174	#else
175	# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176	hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178	#endif
179
180	/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181	# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182	hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186	/*********************************************************************************************************************************
187	* Structures and Typedefs *
188	*********************************************************************************************************************************/
189	/**
190	* VMX transient state.
191	*
192	* A state structure for holding miscellaneous information across
193	* VMX non-root operation and restored after the transition.
194	*/
195	typedef struct VMXTRANSIENT
196	{
197	/** The host's rflags/eflags. */
198	RTCCUINTREG fEFlags;
199	#if HC_ARCH_BITS == 32
200	uint32_t u32Alignment0;
201	#endif
202	/** The guest's TPR value used for TPR shadowing. */
203	uint8_t u8GuestTpr;
204	/** Alignment. */
205	uint8_t abAlignment0[7];
206
207	/** The basic VM-exit reason. */
208	uint16_t uExitReason;
209	/** Alignment. */
210	uint16_t u16Alignment0;
211	/** The VM-exit interruption error code. */
212	uint32_t uExitIntErrorCode;
213	/** The VM-exit exit code qualification. */
214	uint64_t uExitQualification;
215
216	/** The VM-exit interruption-information field. */
217	uint32_t uExitIntInfo;
218	/** The VM-exit instruction-length field. */
219	uint32_t cbInstr;
220	/** The VM-exit instruction-information field. */
221	union
222	{
223	/** Plain unsigned int representation. */
224	uint32_t u;
225	/** INS and OUTS information. */
226	struct
227	{
228	uint32_t u7Reserved0 : 7;
229	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230	uint32_t u3AddrSize : 3;
231	uint32_t u5Reserved1 : 5;
232	/** The segment register (X86_SREG_XXX). */
233	uint32_t iSegReg : 3;
234	uint32_t uReserved2 : 14;
235	} StrIo;
236	/** INVEPT, INVVPID, INVPCID information. */
237	struct
238	{
239	/** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240	uint32_t u2Scaling : 2;
241	uint32_t u5Reserved0 : 5;
242	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243	uint32_t u3AddrSize : 3;
244	uint32_t u1Reserved0 : 1;
245	uint32_t u4Reserved0 : 4;
246	/** The segment register (X86_SREG_XXX). */
247	uint32_t iSegReg : 3;
248	/** The index register (X86_GREG_XXX). */
249	uint32_t iIdxReg : 4;
250	/** Set if index register is invalid. */
251	uint32_t fIdxRegValid : 1;
252	/** The base register (X86_GREG_XXX). */
253	uint32_t iBaseReg : 4;
254	/** Set if base register is invalid. */
255	uint32_t fBaseRegValid : 1;
256	/** Register 2 (X86_GREG_XXX). */
257	uint32_t iReg2 : 4;
258	} Inv;
259	} ExitInstrInfo;
260	/** Whether the VM-entry failed or not. */
261	bool fVMEntryFailed;
262	/** Alignment. */
263	uint8_t abAlignment1[3];
264
265	/** The VM-entry interruption-information field. */
266	uint32_t uEntryIntInfo;
267	/** The VM-entry exception error code field. */
268	uint32_t uEntryXcptErrorCode;
269	/** The VM-entry instruction length field. */
270	uint32_t cbEntryInstr;
271
272	/** IDT-vectoring information field. */
273	uint32_t uIdtVectoringInfo;
274	/** IDT-vectoring error code. */
275	uint32_t uIdtVectoringErrorCode;
276
277	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278	uint32_t fVmcsFieldsRead;
279
280	/** Whether the guest debug state was active at the time of VM-exit. */
281	bool fWasGuestDebugStateActive;
282	/** Whether the hyper debug state was active at the time of VM-exit. */
283	bool fWasHyperDebugStateActive;
284	/** Whether TSC-offsetting should be setup before VM-entry. */
285	bool fUpdateTscOffsettingAndPreemptTimer;
286	/** Whether the VM-exit was caused by a page-fault during delivery of a
287	* contributory exception or a page-fault. */
288	bool fVectoringDoublePF;
289	/** Whether the VM-exit was caused by a page-fault during delivery of an
290	* external interrupt or NMI. */
291	bool fVectoringPF;
292	} VMXTRANSIENT;
293	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298	/** Pointer to VMX transient state. */
299	typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302	/**
303	* MSR-bitmap read permissions.
304	*/
305	typedef enum VMXMSREXITREAD
306	{
307	/** Reading this MSR causes a VM-exit. */
308	VMXMSREXIT_INTERCEPT_READ = 0xb,
309	/** Reading this MSR does not cause a VM-exit. */
310	VMXMSREXIT_PASSTHRU_READ
311	} VMXMSREXITREAD;
312	/** Pointer to MSR-bitmap read permissions. */
313	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315	/**
316	* MSR-bitmap write permissions.
317	*/
318	typedef enum VMXMSREXITWRITE
319	{
320	/** Writing to this MSR causes a VM-exit. */
321	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322	/** Writing to this MSR does not cause a VM-exit. */
323	VMXMSREXIT_PASSTHRU_WRITE
324	} VMXMSREXITWRITE;
325	/** Pointer to MSR-bitmap write permissions. */
326	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329	/**
330	* VMX VM-exit handler.
331	*
332	* @returns Strict VBox status code (i.e. informational status codes too).
333	* @param pVCpu The cross context virtual CPU structure.
334	* @param pVmxTransient Pointer to the VMX-transient structure.
335	*/
336	#ifndef HMVMX_USE_FUNCTION_TABLE
337	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
338	#else
339	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
340	/** Pointer to VM-exit handler. */
341	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342	#endif
343
344	/**
345	* VMX VM-exit handler, non-strict status code.
346	*
347	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
348	*
349	* @returns VBox status code, no informational status code returned.
350	* @param pVCpu The cross context virtual CPU structure.
351	* @param pVmxTransient Pointer to the VMX-transient structure.
352	*
353	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
354	* use of that status code will be replaced with VINF_EM_SOMETHING
355	* later when switching over to IEM.
356	*/
357	#ifndef HMVMX_USE_FUNCTION_TABLE
358	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
359	#else
360	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
361	#endif
362
363
364	/*********************************************************************************************************************************
365	* Internal Functions *
366	*********************************************************************************************************************************/
367	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
368	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
369	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
370	static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
371	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
372	RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
373	#if HC_ARCH_BITS == 32
374	static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
375	#endif
376	#ifndef HMVMX_USE_FUNCTION_TABLE
377	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
378	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
379	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
380	#else
381	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
382	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
383	#endif
384
385
386	/** @name VM-exit handlers.
387	* @{
388	*/
389	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
390	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
391	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
392	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
393	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
394	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
395	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
398	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
399	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
400	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
401	static FNVMXEXITHANDLER hmR0VmxExitHlt;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
403	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
404	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
405	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
406	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
407	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
410	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
411	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
412	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
413	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
415	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
417	static FNVMXEXITHANDLER hmR0VmxExitMwait;
418	static FNVMXEXITHANDLER hmR0VmxExitMtf;
419	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
420	static FNVMXEXITHANDLER hmR0VmxExitPause;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
422	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
423	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
424	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
425	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
426	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
427	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
428	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
429	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
430	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
431	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
432	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
433	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
434	/** @} */
435
436	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
437	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
438	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
440	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
441	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
444
445
446	/*********************************************************************************************************************************
447	* Global Variables *
448	*********************************************************************************************************************************/
449	#ifdef HMVMX_USE_FUNCTION_TABLE
450
451	/**
452	* VMX_EXIT dispatch table.
453	*/
454	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
455	{
456	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
457	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
458	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
459	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
460	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
461	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
462	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
463	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
464	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
465	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
466	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
467	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
468	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
469	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
470	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
471	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
472	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
473	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
474	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
475	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
476	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
477	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
478	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
479	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
480	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
481	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
482	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
483	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
484	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
485	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
486	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
487	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
488	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
489	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
490	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
491	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
492	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
493	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
494	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
495	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
496	/* 40 UNDEFINED */ hmR0VmxExitPause,
497	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
498	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
499	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
500	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
501	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
502	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
503	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
504	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
505	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
506	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
507	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
508	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
509	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
510	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
511	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
512	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
513	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
514	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
515	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
516	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
517	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
518	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
519	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
520	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
521	};
522	#endif /* HMVMX_USE_FUNCTION_TABLE */
523
524	#ifdef VBOX_STRICT
525	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
526	{
527	/* 0 */ "(Not Used)",
528	/* 1 */ "VMCALL executed in VMX root operation.",
529	/* 2 */ "VMCLEAR with invalid physical address.",
530	/* 3 */ "VMCLEAR with VMXON pointer.",
531	/* 4 */ "VMLAUNCH with non-clear VMCS.",
532	/* 5 */ "VMRESUME with non-launched VMCS.",
533	/* 6 */ "VMRESUME after VMXOFF",
534	/* 7 */ "VM-entry with invalid control fields.",
535	/* 8 */ "VM-entry with invalid host state fields.",
536	/* 9 */ "VMPTRLD with invalid physical address.",
537	/* 10 */ "VMPTRLD with VMXON pointer.",
538	/* 11 */ "VMPTRLD with incorrect revision identifier.",
539	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
540	/* 13 */ "VMWRITE to read-only VMCS component.",
541	/* 14 */ "(Not Used)",
542	/* 15 */ "VMXON executed in VMX root operation.",
543	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
544	/* 17 */ "VM-entry with non-launched executing VMCS.",
545	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
546	/* 19 */ "VMCALL with non-clear VMCS.",
547	/* 20 */ "VMCALL with invalid VM-exit control fields.",
548	/* 21 */ "(Not Used)",
549	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
550	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
551	/* 24 */ "VMCALL with invalid SMM-monitor features.",
552	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
553	/* 26 */ "VM-entry with events blocked by MOV SS.",
554	/* 27 */ "(Not Used)",
555	/* 28 */ "Invalid operand to INVEPT/INVVPID."
556	};
557	#endif /* VBOX_STRICT */
558
559
560
561	/**
562	* Updates the VM's last error record.
563	*
564	* If there was a VMX instruction error, reads the error data from the VMCS and
565	* updates VCPU's last error record as well.
566	*
567	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
568	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
569	* VERR_VMX_INVALID_VMCS_FIELD.
570	* @param rc The error code.
571	*/
572	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
573	{
574	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
575	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
576	{
577	AssertPtrReturnVoid(pVCpu);
578	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
579	}
580	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
581	}
582
583
584	/**
585	* Reads the VM-entry interruption-information field from the VMCS into the VMX
586	* transient structure.
587	*
588	* @returns VBox status code.
589	* @param pVmxTransient Pointer to the VMX transient structure.
590	*
591	* @remarks No-long-jump zone!!!
592	*/
593	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
594	{
595	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
596	AssertRCReturn(rc, rc);
597	return VINF_SUCCESS;
598	}
599
600	#ifdef VBOX_STRICT
601	/**
602	* Reads the VM-entry exception error code field from the VMCS into
603	* the VMX transient structure.
604	*
605	* @returns VBox status code.
606	* @param pVmxTransient Pointer to the VMX transient structure.
607	*
608	* @remarks No-long-jump zone!!!
609	*/
610	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
611	{
612	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
613	AssertRCReturn(rc, rc);
614	return VINF_SUCCESS;
615	}
616
617
618	/**
619	* Reads the VM-entry exception error code field from the VMCS into
620	* the VMX transient structure.
621	*
622	* @returns VBox status code.
623	* @param pVmxTransient Pointer to the VMX transient structure.
624	*
625	* @remarks No-long-jump zone!!!
626	*/
627	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628	{
629	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630	AssertRCReturn(rc, rc);
631	return VINF_SUCCESS;
632	}
633	#endif /* VBOX_STRICT */
634
635
636	/**
637	* Reads the VM-exit interruption-information field from the VMCS into the VMX
638	* transient structure.
639	*
640	* @returns VBox status code.
641	* @param pVmxTransient Pointer to the VMX transient structure.
642	*/
643	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644	{
645	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
646	{
647	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648	AssertRCReturn(rc,rc);
649	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
650	}
651	return VINF_SUCCESS;
652	}
653
654
655	/**
656	* Reads the VM-exit interruption error code from the VMCS into the VMX
657	* transient structure.
658	*
659	* @returns VBox status code.
660	* @param pVmxTransient Pointer to the VMX transient structure.
661	*/
662	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663	{
664	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
665	{
666	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667	AssertRCReturn(rc, rc);
668	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
669	}
670	return VINF_SUCCESS;
671	}
672
673
674	/**
675	* Reads the VM-exit instruction length field from the VMCS into the VMX
676	* transient structure.
677	*
678	* @returns VBox status code.
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
684	{
685	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the VM-exit instruction-information field from the VMCS into
695	* the VMX transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*/
700	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701	{
702	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
703	{
704	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705	AssertRCReturn(rc, rc);
706	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
707	}
708	return VINF_SUCCESS;
709	}
710
711
712	/**
713	* Reads the exit code qualification from the VMCS into the VMX transient
714	* structure.
715	*
716	* @returns VBox status code.
717	* @param pVCpu The cross context virtual CPU structure of the
718	* calling EMT. (Required for the VMCS cache case.)
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
724	{
725	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Reads the IDT-vectoring information field from the VMCS into the VMX
735	* transient structure.
736	*
737	* @returns VBox status code.
738	* @param pVmxTransient Pointer to the VMX transient structure.
739	*
740	* @remarks No-long-jump zone!!!
741	*/
742	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743	{
744	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
745	{
746	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
747	AssertRCReturn(rc, rc);
748	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
749	}
750	return VINF_SUCCESS;
751	}
752
753
754	/**
755	* Reads the IDT-vectoring error code from the VMCS into the VMX
756	* transient structure.
757	*
758	* @returns VBox status code.
759	* @param pVmxTransient Pointer to the VMX transient structure.
760	*/
761	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762	{
763	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
764	{
765	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766	AssertRCReturn(rc, rc);
767	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
768	}
769	return VINF_SUCCESS;
770	}
771
772
773	/**
774	* Enters VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	* @param pVM The cross context VM structure. Can be
778	* NULL, after a resume.
779	* @param HCPhysCpuPage Physical address of the VMXON region.
780	* @param pvCpuPage Pointer to the VMXON region.
781	*/
782	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783	{
784	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786	Assert(pvCpuPage);
787	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789	if (pVM)
790	{
791	/* Write the VMCS revision dword to the VMXON region. */
792	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793	}
794
795	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796	RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798	/* Enable the VMX bit in CR4 if necessary. */
799	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801	/* Enter VMX root mode. */
802	int rc = VMXEnable(HCPhysCpuPage);
803	if (RT_FAILURE(rc))
804	{
805	if (!(uOldCr4 & X86_CR4_VMXE))
806	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808	if (pVM)
809	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810	}
811
812	/* Restore interrupts. */
813	ASMSetFlags(fEFlags);
814	return rc;
815	}
816
817
818	/**
819	* Exits VMX root mode operation on the current CPU.
820	*
821	* @returns VBox status code.
822	*/
823	static int hmR0VmxLeaveRootMode(void)
824	{
825	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828	RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830	/* If we're for some reason not in VMX root mode, then don't leave it. */
831	RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833	int rc;
834	if (uHostCR4 & X86_CR4_VMXE)
835	{
836	/* Exit VMX root mode and clear the VMX bit in CR4. */
837	VMXDisable();
838	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839	rc = VINF_SUCCESS;
840	}
841	else
842	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844	/* Restore interrupts. */
845	ASMSetFlags(fEFlags);
846	return rc;
847	}
848
849
850	/**
851	* Allocates and maps one physically contiguous page. The allocated page is
852	* zero'd out. (Used by various VT-x structures).
853	*
854	* @returns IPRT status code.
855	* @param pMemObj Pointer to the ring-0 memory object.
856	* @param ppVirt Where to store the virtual address of the
857	* allocation.
858	* @param pHCPhys Where to store the physical address of the
859	* allocation.
860	*/
861	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862	{
863	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868	if (RT_FAILURE(rc))
869	return rc;
870	ppVirt = RTR0MemObjAddress(pMemObj);
871	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
872	ASMMemZero32(*ppVirt, PAGE_SIZE);
873	return VINF_SUCCESS;
874	}
875
876
877	/**
878	* Frees and unmaps an allocated physical page.
879	*
880	* @param pMemObj Pointer to the ring-0 memory object.
881	* @param ppVirt Where to re-initialize the virtual address of
882	* allocation as 0.
883	* @param pHCPhys Where to re-initialize the physical address of the
884	* allocation as 0.
885	*/
886	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887	{
888	AssertPtr(pMemObj);
889	AssertPtr(ppVirt);
890	AssertPtr(pHCPhys);
891	if (*pMemObj != NIL_RTR0MEMOBJ)
892	{
893	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
894	AssertRC(rc);
895	*pMemObj = NIL_RTR0MEMOBJ;
896	*ppVirt = 0;
897	*pHCPhys = 0;
898	}
899	}
900
901
902	/**
903	* Worker function to free VT-x related structures.
904	*
905	* @returns IPRT status code.
906	* @param pVM The cross context VM structure.
907	*/
908	static void hmR0VmxStructsFree(PVM pVM)
909	{
910	for (VMCPUID i = 0; i < pVM->cCpus; i++)
911	{
912	PVMCPU pVCpu = &pVM->aCpus[i];
913	AssertPtr(pVCpu);
914
915	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922	}
923
924	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927	#endif
928	}
929
930
931	/**
932	* Worker function to allocate VT-x related VM structures.
933	*
934	* @returns IPRT status code.
935	* @param pVM The cross context VM structure.
936	*/
937	static int hmR0VmxStructsAlloc(PVM pVM)
938	{
939	/*
940	* Initialize members up-front so we can cleanup properly on allocation failure.
941	*/
942	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945	pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954	#endif
955	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958	for (VMCPUID i = 0; i < pVM->cCpus; i++)
959	{
960	PVMCPU pVCpu = &pVM->aCpus[i];
961	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965	}
966	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967	#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974	/*
975	* Allocate all the VT-x structures.
976	*/
977	int rc = VINF_SUCCESS;
978	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980	if (RT_FAILURE(rc))
981	goto cleanup;
982	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984	#endif
985
986	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988	{
989	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990	&pVM->hm.s.vmx.HCPhysApicAccess);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	/*
996	* Initialize per-VCPU VT-x structures.
997	*/
998	for (VMCPUID i = 0; i < pVM->cCpus; i++)
999	{
1000	PVMCPU pVCpu = &pVM->aCpus[i];
1001	AssertPtr(pVCpu);
1002
1003	/* Allocate the VM control structure (VMCS). */
1004	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005	if (RT_FAILURE(rc))
1006	goto cleanup;
1007
1008	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1009	if ( PDMHasApic(pVM)
1010	&& (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1011	{
1012	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013	NULL /* pR3Ptr /, NULL / pRCPtr */);
1014	if (RT_FAILURE(rc))
1015	goto cleanup;
1016	}
1017
1018	/*
1019	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020	* transparent accesses of specific MSRs.
1021	*
1022	* If the condition for enabling MSR bitmaps changes here, don't forget to
1023	* update HMAreMsrBitmapsAvailable().
1024	*/
1025	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026	{
1027	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029	if (RT_FAILURE(rc))
1030	goto cleanup;
1031	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032	}
1033
1034	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036	if (RT_FAILURE(rc))
1037	goto cleanup;
1038
1039	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1040	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041	if (RT_FAILURE(rc))
1042	goto cleanup;
1043	}
1044
1045	return VINF_SUCCESS;
1046
1047	cleanup:
1048	hmR0VmxStructsFree(pVM);
1049	return rc;
1050	}
1051
1052
1053	/**
1054	* Does global VT-x initialization (called during module initialization).
1055	*
1056	* @returns VBox status code.
1057	*/
1058	VMMR0DECL(int) VMXR0GlobalInit(void)
1059	{
1060	#ifdef HMVMX_USE_FUNCTION_TABLE
1061	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062	# ifdef VBOX_STRICT
1063	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064	Assert(g_apfnVMExitHandlers[i]);
1065	# endif
1066	#endif
1067	return VINF_SUCCESS;
1068	}
1069
1070
1071	/**
1072	* Does global VT-x termination (called during module termination).
1073	*/
1074	VMMR0DECL(void) VMXR0GlobalTerm()
1075	{
1076	/* Nothing to do currently. */
1077	}
1078
1079
1080	/**
1081	* Sets up and activates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pHostCpu Pointer to the global CPU info struct.
1085	* @param pVM The cross context VM structure. Can be
1086	* NULL after a host resume operation.
1087	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088	* fEnabledByHost is @c true).
1089	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090	* @a fEnabledByHost is @c true).
1091	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092	* enable VT-x on the host.
1093	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1094	*/
1095	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096	void *pvMsrs)
1097	{
1098	Assert(pHostCpu);
1099	Assert(pvMsrs);
1100	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102	/* Enable VT-x if it's not already enabled by the host. */
1103	if (!fEnabledByHost)
1104	{
1105	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106	if (RT_FAILURE(rc))
1107	return rc;
1108	}
1109
1110	/*
1111	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1112	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1113	* invalidated when flushing by VPID.
1114	*/
1115	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117	{
1118	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1119	pHostCpu->fFlushAsidBeforeUse = false;
1120	}
1121	else
1122	pHostCpu->fFlushAsidBeforeUse = true;
1123
1124	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125	++pHostCpu->cTlbFlushes;
1126
1127	return VINF_SUCCESS;
1128	}
1129
1130
1131	/**
1132	* Deactivates VT-x on the current CPU.
1133	*
1134	* @returns VBox status code.
1135	* @param pHostCpu Pointer to the global CPU info struct.
1136	* @param pvCpuPage Pointer to the VMXON region.
1137	* @param HCPhysCpuPage Physical address of the VMXON region.
1138	*
1139	* @remarks This function should never be called when SUPR0EnableVTx() or
1140	* similar was used to enable VT-x on the host.
1141	*/
1142	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143	{
1144	RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1145
1146	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147	return hmR0VmxLeaveRootMode();
1148	}
1149
1150
1151	/**
1152	* Sets the permission bits for the specified MSR in the MSR bitmap.
1153	*
1154	* @param pVCpu The cross context virtual CPU structure.
1155	* @param uMsr The MSR value.
1156	* @param enmRead Whether reading this MSR causes a VM-exit.
1157	* @param enmWrite Whether writing this MSR causes a VM-exit.
1158	*/
1159	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160	{
1161	int32_t iBit;
1162	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164	/*
1165	* Layout:
1166	* 0x000 - 0x3ff - Low MSR read bits
1167	* 0x400 - 0x7ff - High MSR read bits
1168	* 0x800 - 0xbff - Low MSR write bits
1169	* 0xc00 - 0xfff - High MSR write bits
1170	*/
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174	{
1175	iBit = uMsr - UINT32_C(0xC0000000);
1176	pbMsrBitmap += 0x400;
1177	}
1178	else
1179	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181	Assert(iBit <= 0x1fff);
1182	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183	ASMBitSet(pbMsrBitmap, iBit);
1184	else
1185	ASMBitClear(pbMsrBitmap, iBit);
1186
1187	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189	else
1190	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191	}
1192
1193
1194	#ifdef VBOX_STRICT
1195	/**
1196	* Gets the permission bits for the specified MSR in the MSR bitmap.
1197	*
1198	* @returns VBox status code.
1199	* @retval VINF_SUCCESS if the specified MSR is found.
1200	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1201	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202	*
1203	* @param pVCpu The cross context virtual CPU structure.
1204	* @param uMsr The MSR.
1205	* @param penmRead Where to store the read permissions.
1206	* @param penmWrite Where to store the write permissions.
1207	*/
1208	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209	{
1210	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212	int32_t iBit;
1213	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215	/* See hmR0VmxSetMsrPermission() for the layout. */
1216	if (uMsr <= 0x00001FFF)
1217	iBit = uMsr;
1218	else if ( uMsr >= 0xC0000000
1219	&& uMsr <= 0xC0001FFF)
1220	{
1221	iBit = (uMsr - 0xC0000000);
1222	pbMsrBitmap += 0x400;
1223	}
1224	else
1225	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227	Assert(iBit <= 0x1fff);
1228	if (ASMBitTest(pbMsrBitmap, iBit))
1229	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1230	else
1231	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235	else
1236	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237	return VINF_SUCCESS;
1238	}
1239	#endif /* VBOX_STRICT */
1240
1241
1242	/**
1243	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244	* area.
1245	*
1246	* @returns VBox status code.
1247	* @param pVCpu The cross context virtual CPU structure.
1248	* @param cMsrs The number of MSRs.
1249	*/
1250	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251	{
1252	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255	{
1256	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259	}
1260
1261	/* Update number of guest MSRs to load/store across the world-switch. */
1262	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267	AssertRCReturn(rc, rc);
1268
1269	/* Update the VCPU's copy of the MSR count. */
1270	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272	return VINF_SUCCESS;
1273	}
1274
1275
1276	/**
1277	* Adds a new (or updates the value of an existing) guest/host MSR
1278	* pair to be swapped during the world-switch as part of the
1279	* auto-load/store MSR area in the VMCS.
1280	*
1281	* @returns VBox status code.
1282	* @param pVCpu The cross context virtual CPU structure.
1283	* @param uMsr The MSR.
1284	* @param uGuestMsrValue Value of the guest MSR.
1285	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1286	* necessary.
1287	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288	* its value was updated. Optional, can be NULL.
1289	*/
1290	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291	bool *pfAddedAndUpdated)
1292	{
1293	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295	uint32_t i;
1296	for (i = 0; i < cMsrs; i++)
1297	{
1298	if (pGuestMsr->u32Msr == uMsr)
1299	break;
1300	pGuestMsr++;
1301	}
1302
1303	bool fAdded = false;
1304	if (i == cMsrs)
1305	{
1306	++cMsrs;
1307	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314	fAdded = true;
1315	}
1316
1317	/* Update the MSR values in the auto-load/store MSR area. */
1318	pGuestMsr->u32Msr = uMsr;
1319	pGuestMsr->u64Value = uGuestMsrValue;
1320
1321	/* Create/update the MSR slot in the host MSR area. */
1322	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323	pHostMsr += i;
1324	pHostMsr->u32Msr = uMsr;
1325
1326	/*
1327	* Update the host MSR only when requested by the caller AND when we're
1328	* adding it to the auto-load/store area. Otherwise, it would have been
1329	* updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1330	*/
1331	bool fUpdatedMsrValue = false;
1332	if ( fAdded
1333	&& fUpdateHostMsr)
1334	{
1335	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338	fUpdatedMsrValue = true;
1339	}
1340
1341	if (pfAddedAndUpdated)
1342	*pfAddedAndUpdated = fUpdatedMsrValue;
1343	return VINF_SUCCESS;
1344	}
1345
1346
1347	/**
1348	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1349	* auto-load/store MSR area in the VMCS.
1350	*
1351	* @returns VBox status code.
1352	* @param pVCpu The cross context virtual CPU structure.
1353	* @param uMsr The MSR.
1354	*/
1355	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356	{
1357	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359	for (uint32_t i = 0; i < cMsrs; i++)
1360	{
1361	/* Find the MSR. */
1362	if (pGuestMsr->u32Msr == uMsr)
1363	{
1364	/* If it's the last MSR, simply reduce the count. */
1365	if (i == cMsrs - 1)
1366	{
1367	--cMsrs;
1368	break;
1369	}
1370
1371	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373	pLastGuestMsr += cMsrs - 1;
1374	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379	pLastHostMsr += cMsrs - 1;
1380	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381	pHostMsr->u64Value = pLastHostMsr->u64Value;
1382	--cMsrs;
1383	break;
1384	}
1385	pGuestMsr++;
1386	}
1387
1388	/* Update the VMCS if the count changed (meaning the MSR was found). */
1389	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390	{
1391	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392	AssertRCReturn(rc, rc);
1393
1394	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398	Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399	return VINF_SUCCESS;
1400	}
1401
1402	return VERR_NOT_FOUND;
1403	}
1404
1405
1406	/**
1407	* Checks if the specified guest MSR is part of the auto-load/store area in
1408	* the VMCS.
1409	*
1410	* @returns true if found, false otherwise.
1411	* @param pVCpu The cross context virtual CPU structure.
1412	* @param uMsr The MSR to find.
1413	*/
1414	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415	{
1416	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420	{
1421	if (pGuestMsr->u32Msr == uMsr)
1422	return true;
1423	}
1424	return false;
1425	}
1426
1427
1428	/**
1429	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430	*
1431	* @param pVCpu The cross context virtual CPU structure.
1432	*
1433	* @remarks No-long-jump zone!!!
1434	*/
1435	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436	{
1437	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443	{
1444	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446	/*
1447	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449	*/
1450	if (pHostMsr->u32Msr == MSR_K6_EFER)
1451	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452	else
1453	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454	}
1455
1456	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457	}
1458
1459
1460	/**
1461	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1462	* perform lazy restoration of the host MSRs while leaving VT-x.
1463	*
1464	* @param pVCpu The cross context virtual CPU structure.
1465	*
1466	* @remarks No-long-jump zone!!!
1467	*/
1468	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469	{
1470	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472	/*
1473	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474	*/
1475	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476	{
1477	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478	#if HC_ARCH_BITS == 64
1479	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480	{
1481	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485	}
1486	#endif
1487	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1488	}
1489	}
1490
1491
1492	/**
1493	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1494	* lazily while leaving VT-x.
1495	*
1496	* @returns true if it does, false otherwise.
1497	* @param pVCpu The cross context virtual CPU structure.
1498	* @param uMsr The MSR to check.
1499	*/
1500	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501	{
1502	NOREF(pVCpu);
1503	#if HC_ARCH_BITS == 64
1504	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505	{
1506	switch (uMsr)
1507	{
1508	case MSR_K8_LSTAR:
1509	case MSR_K6_STAR:
1510	case MSR_K8_SF_MASK:
1511	case MSR_K8_KERNEL_GS_BASE:
1512	return true;
1513	}
1514	}
1515	#else
1516	RT_NOREF(pVCpu, uMsr);
1517	#endif
1518	return false;
1519	}
1520
1521
1522	/**
1523	* Loads a set of guests MSRs to allow read/passthru to the guest.
1524	*
1525	* The name of this function is slightly confusing. This function does NOT
1526	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1527	* common prefix for functions dealing with "lazy restoration" of the shared
1528	* MSRs.
1529	*
1530	* @param pVCpu The cross context virtual CPU structure.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	*/
1534	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1535	{
1536	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540	#if HC_ARCH_BITS == 64
1541	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542	{
1543	/*
1544	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1545	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1546	* we can skip a few MSR writes.
1547	*
1548	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1549	* guest MSR values in the guest-CPU context might be different to what's currently
1550	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1551	* CPU, see @bugref{8728}.
1552	*/
1553	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1554	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1555	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1556	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1557	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1558	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1559	{
1560	#ifdef VBOX_STRICT
1561	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1562	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1563	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1564	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1565	#endif
1566	}
1567	else
1568	{
1569	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1570	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1571	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1572	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1573	}
1574	}
1575	#endif
1576	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1577	}
1578
1579
1580	/**
1581	* Performs lazy restoration of the set of host MSRs if they were previously
1582	* loaded with guest MSR values.
1583	*
1584	* @param pVCpu The cross context virtual CPU structure.
1585	*
1586	* @remarks No-long-jump zone!!!
1587	* @remarks The guest MSRs should have been saved back into the guest-CPU
1588	* context by hmR0VmxImportGuestState()!!!
1589	*/
1590	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1591	{
1592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1594
1595	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1596	{
1597	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1598	#if HC_ARCH_BITS == 64
1599	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1600	{
1601	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1602	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1603	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1604	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1605	}
1606	#endif
1607	}
1608	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1609	}
1610
1611
1612	/**
1613	* Verifies that our cached values of the VMCS fields are all consistent with
1614	* what's actually present in the VMCS.
1615	*
1616	* @returns VBox status code.
1617	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1618	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1619	* VMCS content. HMCPU error-field is
1620	* updated, see VMX_VCI_XXX.
1621	* @param pVCpu The cross context virtual CPU structure.
1622	*/
1623	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1624	{
1625	uint32_t u32Val;
1626	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1627	AssertRCReturn(rc, rc);
1628	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1629	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1630	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1631	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1632
1633	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1634	AssertRCReturn(rc, rc);
1635	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1636	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1637	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1638	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1639
1640	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1641	AssertRCReturn(rc, rc);
1642	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1643	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1645	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1646
1647	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1648	AssertRCReturn(rc, rc);
1649	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1650	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1651	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1652	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1653
1654	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1655	{
1656	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1657	AssertRCReturn(rc, rc);
1658	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1659	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1660	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1661	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1662	}
1663
1664	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1665	AssertRCReturn(rc, rc);
1666	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1667	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1668	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1669	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1670
1671	uint64_t u64Val;
1672	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1673	AssertRCReturn(rc, rc);
1674	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1675	("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1676	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1677	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1678
1679	return VINF_SUCCESS;
1680	}
1681
1682
1683	#ifdef VBOX_STRICT
1684	/**
1685	* Verifies that our cached host EFER value has not changed
1686	* since we cached it.
1687	*
1688	* @param pVCpu The cross context virtual CPU structure.
1689	*/
1690	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1691	{
1692	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1693
1694	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1695	{
1696	uint64_t u64Val;
1697	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1698	AssertRC(rc);
1699
1700	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1701	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1702	}
1703	}
1704
1705
1706	/**
1707	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1708	* VMCS are correct.
1709	*
1710	* @param pVCpu The cross context virtual CPU structure.
1711	*/
1712	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1713	{
1714	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1715
1716	/* Verify MSR counts in the VMCS are what we think it should be. */
1717	uint32_t cMsrs;
1718	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1719	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1720
1721	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1722	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1723
1724	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1725	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1726
1727	PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1728	PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1729	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1730	{
1731	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1732	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1733	pGuestMsr->u32Msr, cMsrs));
1734
1735	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1736	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1737	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1738
1739	/* Verify that the permissions are as expected in the MSR bitmap. */
1740	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1741	{
1742	VMXMSREXITREAD enmRead;
1743	VMXMSREXITWRITE enmWrite;
1744	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1745	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1746	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1747	{
1748	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1749	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1750	}
1751	else
1752	{
1753	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1754	pGuestMsr->u32Msr, cMsrs));
1755	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1756	pGuestMsr->u32Msr, cMsrs));
1757	}
1758	}
1759	}
1760	}
1761	#endif /* VBOX_STRICT */
1762
1763
1764	/**
1765	* Flushes the TLB using EPT.
1766	*
1767	* @returns VBox status code.
1768	* @param pVCpu The cross context virtual CPU structure of the calling
1769	* EMT. Can be NULL depending on @a enmTlbFlush.
1770	* @param enmTlbFlush Type of flush.
1771	*
1772	* @remarks Caller is responsible for making sure this function is called only
1773	* when NestedPaging is supported and providing @a enmTlbFlush that is
1774	* supported by the CPU.
1775	* @remarks Can be called with interrupts disabled.
1776	*/
1777	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1778	{
1779	uint64_t au64Descriptor[2];
1780	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1781	au64Descriptor[0] = 0;
1782	else
1783	{
1784	Assert(pVCpu);
1785	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1786	}
1787	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1788
1789	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1790	AssertMsg(rc == VINF_SUCCESS,
1791	("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1792
1793	if ( RT_SUCCESS(rc)
1794	&& pVCpu)
1795	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1796	}
1797
1798
1799	/**
1800	* Flushes the TLB using VPID.
1801	*
1802	* @returns VBox status code.
1803	* @param pVCpu The cross context virtual CPU structure of the calling
1804	* EMT. Can be NULL depending on @a enmTlbFlush.
1805	* @param enmTlbFlush Type of flush.
1806	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1807	* on @a enmTlbFlush).
1808	*
1809	* @remarks Can be called with interrupts disabled.
1810	*/
1811	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1812	{
1813	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1814
1815	uint64_t au64Descriptor[2];
1816	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1817	{
1818	au64Descriptor[0] = 0;
1819	au64Descriptor[1] = 0;
1820	}
1821	else
1822	{
1823	AssertPtr(pVCpu);
1824	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827	au64Descriptor[1] = GCPtr;
1828	}
1829
1830	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1831	AssertMsg(rc == VINF_SUCCESS,
1832	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833
1834	if ( RT_SUCCESS(rc)
1835	&& pVCpu)
1836	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837	NOREF(rc);
1838	}
1839
1840
1841	/**
1842	* Invalidates a guest page by guest virtual address. Only relevant for
1843	* EPT/VPID, otherwise there is nothing really to invalidate.
1844	*
1845	* @returns VBox status code.
1846	* @param pVCpu The cross context virtual CPU structure.
1847	* @param GCVirt Guest virtual address of the page to invalidate.
1848	*/
1849	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1850	{
1851	AssertPtr(pVCpu);
1852	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1853
1854	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1855	if (!fFlushPending)
1856	{
1857	/*
1858	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1859	* the EPT case. See @bugref{6043} and @bugref{6177}.
1860	*
1861	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1862	* as this function maybe called in a loop with individual addresses.
1863	*/
1864	PVM pVM = pVCpu->CTX_SUFF(pVM);
1865	if (pVM->hm.s.vmx.fVpid)
1866	{
1867	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1868
1869	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1870	/*
1871	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1872	* where executing INVVPID outside 64-bit mode does not flush translations of
1873	* 64-bit linear addresses, see @bugref{6208#c72}.
1874	*/
1875	if (RT_HI_U32(GCVirt))
1876	fVpidFlush = false;
1877	#endif
1878
1879	if (fVpidFlush)
1880	{
1881	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1882	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1883	}
1884	else
1885	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1886	}
1887	else if (pVM->hm.s.fNestedPaging)
1888	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1889	}
1890
1891	return VINF_SUCCESS;
1892	}
1893
1894
1895	/**
1896	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1897	* case where neither EPT nor VPID is supported by the CPU.
1898	*
1899	* @param pVCpu The cross context virtual CPU structure.
1900	* @param pCpu Pointer to the global HM struct.
1901	*
1902	* @remarks Called with interrupts disabled.
1903	*/
1904	static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1905	{
1906	AssertPtr(pVCpu);
1907	AssertPtr(pCpu);
1908
1909	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1910
1911	Assert(pCpu->idCpu != NIL_RTCPUID);
1912	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1913	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1914	pVCpu->hm.s.fForceTLBFlush = false;
1915	return;
1916	}
1917
1918
1919	/**
1920	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1921	*
1922	* @param pVCpu The cross context virtual CPU structure.
1923	* @param pCpu Pointer to the global HM CPU struct.
1924	*
1925	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1926	* nomenclature. The reason is, to avoid confusion in compare statements
1927	* since the host-CPU copies are named "ASID".
1928	*
1929	* @remarks Called with interrupts disabled.
1930	*/
1931	static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1932	{
1933	#ifdef VBOX_WITH_STATISTICS
1934	bool fTlbFlushed = false;
1935	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1936	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1937	if (!fTlbFlushed) \
1938	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1939	} while (0)
1940	#else
1941	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1942	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1943	#endif
1944
1945	AssertPtr(pCpu);
1946	AssertPtr(pVCpu);
1947	Assert(pCpu->idCpu != NIL_RTCPUID);
1948
1949	PVM pVM = pVCpu->CTX_SUFF(pVM);
1950	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1951	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1952	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1953
1954	/*
1955	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
1956	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1957	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1958	* cannot reuse the current ASID anymore.
1959	*/
1960	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1961	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1962	{
1963	++pCpu->uCurrentAsid;
1964	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1965	{
1966	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1967	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1968	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1969	}
1970
1971	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1972	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1973	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1974
1975	/*
1976	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1977	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1978	*/
1979	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1980	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1981	HMVMX_SET_TAGGED_TLB_FLUSHED();
1982	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1983	}
1984
1985	/* Check for explicit TLB flushes. */
1986	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1987	{
1988	/*
1989	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1990	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1991	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1992	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1993	* mappings, see @bugref{6568}.
1994	*
1995	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1996	*/
1997	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1998	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1999	HMVMX_SET_TAGGED_TLB_FLUSHED();
2000	}
2001
2002	pVCpu->hm.s.fForceTLBFlush = false;
2003	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2004
2005	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2006	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2007	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2008	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2009	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2010	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2011	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2012	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2013	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2014
2015	/* Update VMCS with the VPID. */
2016	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2017	AssertRC(rc);
2018
2019	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2020	}
2021
2022
2023	/**
2024	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2025	*
2026	* @returns VBox status code.
2027	* @param pVCpu The cross context virtual CPU structure.
2028	* @param pCpu Pointer to the global HM CPU struct.
2029	*
2030	* @remarks Called with interrupts disabled.
2031	*/
2032	static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2033	{
2034	AssertPtr(pVCpu);
2035	AssertPtr(pCpu);
2036	Assert(pCpu->idCpu != NIL_RTCPUID);
2037	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2038	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2039
2040	/*
2041	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2042	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2043	*/
2044	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2045	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2046	{
2047	pVCpu->hm.s.fForceTLBFlush = true;
2048	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2049	}
2050
2051	/* Check for explicit TLB flushes. */
2052	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2053	{
2054	pVCpu->hm.s.fForceTLBFlush = true;
2055	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2056	}
2057
2058	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2059	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2060
2061	if (pVCpu->hm.s.fForceTLBFlush)
2062	{
2063	hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2064	pVCpu->hm.s.fForceTLBFlush = false;
2065	}
2066	}
2067
2068
2069	/**
2070	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2071	*
2072	* @returns VBox status code.
2073	* @param pVCpu The cross context virtual CPU structure.
2074	* @param pCpu Pointer to the global HM CPU struct.
2075	*
2076	* @remarks Called with interrupts disabled.
2077	*/
2078	static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2079	{
2080	AssertPtr(pVCpu);
2081	AssertPtr(pCpu);
2082	Assert(pCpu->idCpu != NIL_RTCPUID);
2083	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2084	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2085
2086	/*
2087	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2088	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2089	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2090	* cannot reuse the current ASID anymore.
2091	*/
2092	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2093	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2094	{
2095	pVCpu->hm.s.fForceTLBFlush = true;
2096	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2097	}
2098
2099	/* Check for explicit TLB flushes. */
2100	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2101	{
2102	/*
2103	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2104	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2105	* fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2106	* include fExplicitFlush's too) - an obscure corner case.
2107	*/
2108	pVCpu->hm.s.fForceTLBFlush = true;
2109	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2110	}
2111
2112	PVM pVM = pVCpu->CTX_SUFF(pVM);
2113	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2114	if (pVCpu->hm.s.fForceTLBFlush)
2115	{
2116	++pCpu->uCurrentAsid;
2117	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2118	{
2119	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2120	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2121	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2122	}
2123
2124	pVCpu->hm.s.fForceTLBFlush = false;
2125	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2126	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2127	if (pCpu->fFlushAsidBeforeUse)
2128	{
2129	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2130	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2131	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2132	{
2133	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2134	pCpu->fFlushAsidBeforeUse = false;
2135	}
2136	else
2137	{
2138	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2139	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2140	}
2141	}
2142	}
2143
2144	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2145	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2146	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2147	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2148	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2149	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2150	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2151
2152	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2153	AssertRC(rc);
2154	}
2155
2156
2157	/**
2158	* Flushes the guest TLB entry based on CPU capabilities.
2159	*
2160	* @param pVCpu The cross context virtual CPU structure.
2161	* @param pCpu Pointer to the global HM CPU struct.
2162	*/
2163	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2164	{
2165	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2166	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2167	#endif
2168	PVM pVM = pVCpu->CTX_SUFF(pVM);
2169	switch (pVM->hm.s.vmx.enmTlbFlushType)
2170	{
2171	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2172	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2173	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2174	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2175	default:
2176	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2177	break;
2178	}
2179	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2180	}
2181
2182
2183	/**
2184	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2185	* TLB entries from the host TLB before VM-entry.
2186	*
2187	* @returns VBox status code.
2188	* @param pVM The cross context VM structure.
2189	*/
2190	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2191	{
2192	/*
2193	* Determine optimal flush type for Nested Paging.
2194	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2195	* guest execution (see hmR3InitFinalizeR0()).
2196	*/
2197	if (pVM->hm.s.fNestedPaging)
2198	{
2199	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2200	{
2201	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2202	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2203	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2204	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2205	else
2206	{
2207	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2208	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2209	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2210	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2211	}
2212
2213	/* Make sure the write-back cacheable memory type for EPT is supported. */
2214	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2215	{
2216	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2217	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2218	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2219	}
2220
2221	/* EPT requires a page-walk length of 4. */
2222	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2223	{
2224	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2225	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2226	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2227	}
2228	}
2229	else
2230	{
2231	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2232	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2233	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2234	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2235	}
2236	}
2237
2238	/*
2239	* Determine optimal flush type for VPID.
2240	*/
2241	if (pVM->hm.s.vmx.fVpid)
2242	{
2243	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2244	{
2245	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2246	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2247	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2248	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2249	else
2250	{
2251	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2252	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2253	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2254	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2255	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2256	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2257	pVM->hm.s.vmx.fVpid = false;
2258	}
2259	}
2260	else
2261	{
2262	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2263	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2264	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2265	pVM->hm.s.vmx.fVpid = false;
2266	}
2267	}
2268
2269	/*
2270	* Setup the handler for flushing tagged-TLBs.
2271	*/
2272	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2273	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2274	else if (pVM->hm.s.fNestedPaging)
2275	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2276	else if (pVM->hm.s.vmx.fVpid)
2277	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2278	else
2279	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2280	return VINF_SUCCESS;
2281	}
2282
2283
2284	/**
2285	* Sets up pin-based VM-execution controls in the VMCS.
2286	*
2287	* @returns VBox status code.
2288	* @param pVCpu The cross context virtual CPU structure.
2289	*
2290	* @remarks We don't really care about optimizing vmwrites here as it's done only
2291	* once per VM and hence we don't care about VMCS-field cache comparisons.
2292	*/
2293	static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2294	{
2295	PVM pVM = pVCpu->CTX_SUFF(pVM);
2296	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2297	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2298
2299	fVal \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2300	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2301
2302	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2303	fVal \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2304
2305	/* Enable the VMX preemption timer. */
2306	if (pVM->hm.s.vmx.fUsePreemptTimer)
2307	{
2308	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2309	fVal \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2310	}
2311
2312	#if 0
2313	/* Enable posted-interrupt processing. */
2314	if (pVM->hm.s.fPostedIntrs)
2315	{
2316	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2317	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2318	fVal \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2319	}
2320	#endif
2321
2322	if ((fVal & fZap) != fVal)
2323	{
2324	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2325	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2326	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2327	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2328	}
2329
2330	/* Commit it to the VMCS and update our cache. */
2331	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2332	AssertRCReturn(rc, rc);
2333	pVCpu->hm.s.vmx.u32PinCtls = fVal;
2334
2335	return VINF_SUCCESS;
2336	}
2337
2338
2339	/**
2340	* Sets up secondary processor-based VM-execution controls in the VMCS.
2341	*
2342	* @returns VBox status code.
2343	* @param pVCpu The cross context virtual CPU structure.
2344	*
2345	* @remarks We don't really care about optimizing vmwrites here as it's done only
2346	* once per VM and hence we don't care about VMCS-field cache comparisons.
2347	*/
2348	static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2349	{
2350	PVM pVM = pVCpu->CTX_SUFF(pVM);
2351	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2352	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2353
2354	/* WBINVD causes a VM-exit. */
2355	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2356	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2357
2358	/* Enable EPT (aka nested-paging). */
2359	if (pVM->hm.s.fNestedPaging)
2360	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2361
2362	/*
2363	* Enable the INVPCID instruction if supported by the hardware and we expose
2364	* it to the guest. Without this, guest executing INVPCID would cause a #UD.
2365	*/
2366	if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2367	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2368	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2369
2370	/* Enable VPID. */
2371	if (pVM->hm.s.vmx.fVpid)
2372	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2373
2374	/* Enable Unrestricted guest execution. */
2375	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2376	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2377
2378	#if 0
2379	if (pVM->hm.s.fVirtApicRegs)
2380	{
2381	/* Enable APIC-register virtualization. */
2382	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2383	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2384
2385	/* Enable virtual-interrupt delivery. */
2386	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2387	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2388	}
2389	#endif
2390
2391	/* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2392	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2393	* done dynamically. */
2394	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2395	{
2396	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2397	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2398	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2399	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2400	AssertRCReturn(rc, rc);
2401	}
2402
2403	/* Enable RDTSCP. */
2404	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2405	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2406
2407	/* Enable Pause-Loop exiting. */
2408	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2409	&& pVM->hm.s.vmx.cPleGapTicks
2410	&& pVM->hm.s.vmx.cPleWindowTicks)
2411	{
2412	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2413
2414	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2415	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2416	AssertRCReturn(rc, rc);
2417	}
2418
2419	if ((fVal & fZap) != fVal)
2420	{
2421	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2422	pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2423	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2424	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2425	}
2426
2427	/* Commit it to the VMCS and update our cache. */
2428	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2429	AssertRCReturn(rc, rc);
2430	pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2431
2432	return VINF_SUCCESS;
2433	}
2434
2435
2436	/**
2437	* Sets up processor-based VM-execution controls in the VMCS.
2438	*
2439	* @returns VBox status code.
2440	* @param pVCpu The cross context virtual CPU structure.
2441	*
2442	* @remarks We don't really care about optimizing vmwrites here as it's done only
2443	* once per VM and hence we don't care about VMCS-field cache comparisons.
2444	*/
2445	static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2446	{
2447	PVM pVM = pVCpu->CTX_SUFF(pVM);
2448	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2449	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2450
2451	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2452	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2453	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2454	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2455	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2456	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2457	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2458
2459	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2460	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2461	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2462	{
2463	LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2464	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2465	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2466	}
2467
2468	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2469	if (!pVM->hm.s.fNestedPaging)
2470	{
2471	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2472	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2473	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2474	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2475	}
2476
2477	/* Use TPR shadowing if supported by the CPU. */
2478	if ( PDMHasApic(pVM)
2479	&& pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2480	{
2481	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2482	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2483	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2484	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2485	AssertRCReturn(rc, rc);
2486
2487	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2488	/* CR8 writes cause a VM-exit based on TPR threshold. */
2489	Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2490	Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2491	}
2492	else
2493	{
2494	/*
2495	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2496	* Set this control only for 64-bit guests.
2497	*/
2498	if (pVM->hm.s.fAllow64BitGuests)
2499	{
2500	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2501	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2502	}
2503	}
2504
2505	/* Use MSR-bitmaps if supported by the CPU. */
2506	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2507	{
2508	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2509
2510	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2511	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2512	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2513	AssertRCReturn(rc, rc);
2514
2515	/*
2516	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2517	* automatically using dedicated fields in the VMCS.
2518	*/
2519	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2520	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2521	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2522	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2523	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2524	#if HC_ARCH_BITS == 64
2525	/*
2526	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2527	*/
2528	if (pVM->hm.s.fAllow64BitGuests)
2529	{
2530	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2533	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2534	}
2535	#endif
2536	/*
2537	* The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2538	* access (writes need to be executed without exiting, reds will #GP-fault anyway).
2539	*/
2540	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2541	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2542
2543	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2544	}
2545
2546	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2547	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2548	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2549
2550	if ((fVal & fZap) != fVal)
2551	{
2552	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2553	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2554	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2555	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556	}
2557
2558	/* Commit it to the VMCS and update our cache. */
2559	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2560	AssertRCReturn(rc, rc);
2561	pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2562
2563	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2564	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2565	return hmR0VmxSetupProcCtls2(pVCpu);
2566
2567	/* Sanity check, should not really happen. */
2568	if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2569	{
2570	LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2571	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2572	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2573	}
2574
2575	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2576	return VINF_SUCCESS;
2577	}
2578
2579
2580	/**
2581	* Sets up miscellaneous (everything other than Pin & Processor-based
2582	* VM-execution) control fields in the VMCS.
2583	*
2584	* @returns VBox status code.
2585	* @param pVCpu The cross context virtual CPU structure.
2586	*/
2587	static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2588	{
2589	AssertPtr(pVCpu);
2590
2591	int rc = VERR_GENERAL_FAILURE;
2592
2593	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2594	#if 0
2595	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2596	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2597	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2598
2599	/*
2600	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2601	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2602	* We thus use the exception bitmap to control it rather than use both.
2603	*/
2604	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2605	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2606
2607	/* All IO & IOIO instructions cause VM-exits. */
2608	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2609	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2610
2611	/* Initialize the MSR-bitmap area. */
2612	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2613	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2614	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2615	AssertRCReturn(rc, rc);
2616	#endif
2617
2618	/* Setup MSR auto-load/store area. */
2619	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2620	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2621	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2622	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2623	AssertRCReturn(rc, rc);
2624
2625	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2626	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2627	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2628	AssertRCReturn(rc, rc);
2629
2630	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2631	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2632	AssertRCReturn(rc, rc);
2633
2634	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2635	#if 0
2636	/* Setup debug controls */
2637	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2638	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2639	AssertRCReturn(rc, rc);
2640	#endif
2641
2642	return rc;
2643	}
2644
2645
2646	/**
2647	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2648	*
2649	* We shall setup those exception intercepts that don't change during the
2650	* lifetime of the VM here. The rest are done dynamically while loading the
2651	* guest state.
2652	*
2653	* @returns VBox status code.
2654	* @param pVCpu The cross context virtual CPU structure.
2655	*/
2656	static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2657	{
2658	AssertPtr(pVCpu);
2659
2660	uint32_t uXcptBitmap;
2661
2662	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2663	uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2664
2665	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2666	and writes, and because recursive #DBs can cause the CPU hang, we must always
2667	intercept #DB. */
2668	uXcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2669
2670	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2671	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2672	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
2673
2674	/* Commit it to the VMCS. */
2675	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2676	AssertRCReturn(rc, rc);
2677
2678	/* Update our cache of the exception bitmap. */
2679	pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2680	return VINF_SUCCESS;
2681	}
2682
2683
2684	/**
2685	* Does per-VM VT-x initialization.
2686	*
2687	* @returns VBox status code.
2688	* @param pVM The cross context VM structure.
2689	*/
2690	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691	{
2692	LogFlowFunc(("pVM=%p\n", pVM));
2693
2694	int rc = hmR0VmxStructsAlloc(pVM);
2695	if (RT_FAILURE(rc))
2696	{
2697	LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698	return rc;
2699	}
2700
2701	return VINF_SUCCESS;
2702	}
2703
2704
2705	/**
2706	* Does per-VM VT-x termination.
2707	*
2708	* @returns VBox status code.
2709	* @param pVM The cross context VM structure.
2710	*/
2711	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712	{
2713	LogFlowFunc(("pVM=%p\n", pVM));
2714
2715	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718	#endif
2719	hmR0VmxStructsFree(pVM);
2720	return VINF_SUCCESS;
2721	}
2722
2723
2724	/**
2725	* Sets up the VM for execution under VT-x.
2726	* This function is only called once per-VM during initialization.
2727	*
2728	* @returns VBox status code.
2729	* @param pVM The cross context VM structure.
2730	*/
2731	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732	{
2733	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736	LogFlowFunc(("pVM=%p\n", pVM));
2737
2738	/*
2739	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be
2740	* allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2741	* pRealModeTSS, see hmR3InitFinalizeR0Intel().
2742	*/
2743	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2744	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2745	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2746	{
2747	LogRelFunc(("Invalid real-on-v86 state.\n"));
2748	return VERR_INTERNAL_ERROR;
2749	}
2750
2751	/* Initialize these always, see hmR3InitFinalizeR0().*/
2752	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2753	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2754
2755	/* Setup the tagged-TLB flush handlers. */
2756	int rc = hmR0VmxSetupTaggedTlb(pVM);
2757	if (RT_FAILURE(rc))
2758	{
2759	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2760	return rc;
2761	}
2762
2763	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2764	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2765	#if HC_ARCH_BITS == 64
2766	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2767	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2768	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2769	{
2770	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2771	}
2772	#endif
2773
2774	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2775	RTCCUINTREG uHostCR4 = ASMGetCR4();
2776	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2777	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2778
2779	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2780	{
2781	PVMCPU pVCpu = &pVM->aCpus[i];
2782	AssertPtr(pVCpu);
2783	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2784
2785	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2786	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2787
2788	/* Set revision dword at the beginning of the VMCS structure. */
2789	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2790
2791	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2792	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2793	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2794	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2795
2796	/* Load this VMCS as the current VMCS. */
2797	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2798	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2799	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2800
2801	rc = hmR0VmxSetupPinCtls(pVCpu);
2802	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2803	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2804
2805	rc = hmR0VmxSetupProcCtls(pVCpu);
2806	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2807	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2808
2809	rc = hmR0VmxSetupMiscCtls(pVCpu);
2810	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2811	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2812
2813	rc = hmR0VmxInitXcptBitmap(pVCpu);
2814	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2815	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2816
2817	#if HC_ARCH_BITS == 32
2818	rc = hmR0VmxInitVmcsReadCache(pVCpu);
2819	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2820	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2821	#endif
2822
2823	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2824	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2825	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2826	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2827
2828	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2829
2830	hmR0VmxUpdateErrorRecord(pVCpu, rc);
2831	}
2832
2833	return VINF_SUCCESS;
2834	}
2835
2836
2837	/**
2838	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2839	* the VMCS.
2840	*
2841	* @returns VBox status code.
2842	*/
2843	static int hmR0VmxExportHostControlRegs(void)
2844	{
2845	RTCCUINTREG uReg = ASMGetCR0();
2846	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2847	AssertRCReturn(rc, rc);
2848
2849	uReg = ASMGetCR3();
2850	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2851	AssertRCReturn(rc, rc);
2852
2853	uReg = ASMGetCR4();
2854	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2855	AssertRCReturn(rc, rc);
2856	return rc;
2857	}
2858
2859
2860	/**
2861	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2862	* the host-state area in the VMCS.
2863	*
2864	* @returns VBox status code.
2865	* @param pVCpu The cross context virtual CPU structure.
2866	*/
2867	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2868	{
2869	#if HC_ARCH_BITS == 64
2870	/**
2871	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2872	* requirements. See hmR0VmxExportHostSegmentRegs().
2873	*/
2874	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2875	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2876	{ \
2877	bool fValidSelector = true; \
2878	if ((selValue) & X86_SEL_LDT) \
2879	{ \
2880	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2881	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2882	} \
2883	if (fValidSelector) \
2884	{ \
2885	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2886	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2887	} \
2888	(selValue) = 0; \
2889	}
2890
2891	/*
2892	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2893	* should -not- save the messed up state without restoring the original host-state,
2894	* see @bugref{7240}.
2895	*
2896	* This apparently can happen (most likely the FPU changes), deal with it rather than
2897	* asserting. Was observed booting Solaris 10u10 32-bit guest.
2898	*/
2899	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2900	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2901	{
2902	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2903	pVCpu->idCpu));
2904	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2905	}
2906	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2907	#else
2908	RT_NOREF(pVCpu);
2909	#endif
2910
2911	/*
2912	* Host DS, ES, FS and GS segment registers.
2913	*/
2914	#if HC_ARCH_BITS == 64
2915	RTSEL uSelDS = ASMGetDS();
2916	RTSEL uSelES = ASMGetES();
2917	RTSEL uSelFS = ASMGetFS();
2918	RTSEL uSelGS = ASMGetGS();
2919	#else
2920	RTSEL uSelDS = 0;
2921	RTSEL uSelES = 0;
2922	RTSEL uSelFS = 0;
2923	RTSEL uSelGS = 0;
2924	#endif
2925
2926	/*
2927	* Host CS and SS segment registers.
2928	*/
2929	RTSEL uSelCS = ASMGetCS();
2930	RTSEL uSelSS = ASMGetSS();
2931
2932	/*
2933	* Host TR segment register.
2934	*/
2935	RTSEL uSelTR = ASMGetTR();
2936
2937	#if HC_ARCH_BITS == 64
2938	/*
2939	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2940	* gain VM-entry and restore them before we get preempted.
2941	*
2942	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2943	*/
2944	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2945	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2946	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2947	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2948	# undef VMXLOCAL_ADJUST_HOST_SEG
2949	#endif
2950
2951	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2952	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2953	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2954	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2955	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2956	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2957	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2958	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2959	Assert(uSelCS);
2960	Assert(uSelTR);
2961
2962	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2963	#if 0
2964	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2965	Assert(uSelSS != 0);
2966	#endif
2967
2968	/* Write these host selector fields into the host-state area in the VMCS. */
2969	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2970	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2971	#if HC_ARCH_BITS == 64
2972	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2973	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2974	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2975	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2976	#else
2977	NOREF(uSelDS);
2978	NOREF(uSelES);
2979	NOREF(uSelFS);
2980	NOREF(uSelGS);
2981	#endif
2982	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2983	AssertRCReturn(rc, rc);
2984
2985	/*
2986	* Host GDTR and IDTR.
2987	*/
2988	RTGDTR Gdtr;
2989	RTIDTR Idtr;
2990	RT_ZERO(Gdtr);
2991	RT_ZERO(Idtr);
2992	ASMGetGDTR(&Gdtr);
2993	ASMGetIDTR(&Idtr);
2994	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2995	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2996	AssertRCReturn(rc, rc);
2997
2998	#if HC_ARCH_BITS == 64
2999	/*
3000	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3001	* them to the maximum limit (0xffff) on every VM-exit.
3002	*/
3003	if (Gdtr.cbGdt != 0xffff)
3004	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3005
3006	/*
3007	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3008	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3009	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3010	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3011	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3012	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3013	* at 0xffff on hosts where we are sure it won't cause trouble.
3014	*/
3015	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3016	if (Idtr.cbIdt < 0x0fff)
3017	# else
3018	if (Idtr.cbIdt != 0xffff)
3019	# endif
3020	{
3021	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3022	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3023	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3024	}
3025	#endif
3026
3027	/*
3028	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3029	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3030	* RPL should be too in most cases.
3031	*/
3032	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3033	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3034
3035	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3036	#if HC_ARCH_BITS == 64
3037	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3038
3039	/*
3040	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3041	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3042	* restoration if the host has something else. Task switching is not supported in 64-bit
3043	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3044	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3045	*
3046	* [1] See Intel spec. 3.5 "System Descriptor Types".
3047	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3048	*/
3049	PVM pVM = pVCpu->CTX_SUFF(pVM);
3050	Assert(pDesc->System.u4Type == 11);
3051	if ( pDesc->System.u16LimitLow != 0x67
3052	\|\| pDesc->System.u4LimitHigh)
3053	{
3054	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3055	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3056	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3057	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3058	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3059	}
3060
3061	/*
3062	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3063	*/
3064	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
3065	{
3066	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3067	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3068	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3069	{
3070	/* The GDT is read-only but the writable GDT is available. */
3071	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3072	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3073	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3074	AssertRCReturn(rc, rc);
3075	}
3076	}
3077	#else
3078	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079	#endif
3080	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081	AssertRCReturn(rc, rc);
3082
3083	/*
3084	* Host FS base and GS base.
3085	*/
3086	#if HC_ARCH_BITS == 64
3087	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091	AssertRCReturn(rc, rc);
3092
3093	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098	#endif
3099	return VINF_SUCCESS;
3100	}
3101
3102
3103	/**
3104	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
3105	* host-state area of the VMCS.
3106	*
3107	* Theses MSRs will be automatically restored on the host after every successful
3108	* VM-exit.
3109	*
3110	* @returns VBox status code.
3111	* @param pVCpu The cross context virtual CPU structure.
3112	*
3113	* @remarks No-long-jump zone!!!
3114	*/
3115	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3116	{
3117	AssertPtr(pVCpu);
3118	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3119
3120	/*
3121	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3122	* rather than swapping them on every VM-entry.
3123	*/
3124	hmR0VmxLazySaveHostMsrs(pVCpu);
3125
3126	/*
3127	* Host Sysenter MSRs.
3128	*/
3129	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3130	#if HC_ARCH_BITS == 32
3131	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3132	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3133	#else
3134	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3135	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3136	#endif
3137	AssertRCReturn(rc, rc);
3138
3139	/*
3140	* Host EFER MSR.
3141	*
3142	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3143	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3144	*/
3145	PVM pVM = pVCpu->CTX_SUFF(pVM);
3146	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3147	{
3148	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3149	AssertRCReturn(rc, rc);
3150	}
3151
3152	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3153
3154	return VINF_SUCCESS;
3155	}
3156
3157
3158	/**
3159	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3160	*
3161	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3162	* these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3163	* hmR0VMxExportGuestEntryCtls().
3164	*
3165	* @returns true if we need to load guest EFER, false otherwise.
3166	* @param pVCpu The cross context virtual CPU structure.
3167	*
3168	* @remarks Requires EFER, CR4.
3169	* @remarks No-long-jump zone!!!
3170	*/
3171	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3172	{
3173	#ifdef HMVMX_ALWAYS_SWAP_EFER
3174	return true;
3175	#endif
3176
3177	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3178	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3179	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3180	if (CPUMIsGuestInLongModeEx(pCtx))
3181	return false;
3182	#endif
3183
3184	PVM pVM = pVCpu->CTX_SUFF(pVM);
3185	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3186	uint64_t const u64GuestEfer = pCtx->msrEFER;
3187
3188	/*
3189	* For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3190	* guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3191	*/
3192	if ( CPUMIsGuestInLongModeEx(pCtx)
3193	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3194	{
3195	return true;
3196	}
3197
3198	/*
3199	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3200	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3201	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3202	*/
3203	if ( (pCtx->cr4 & X86_CR4_PAE)
3204	&& (pCtx->cr0 & X86_CR0_PG)
3205	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3206	{
3207	/* Assert that host is PAE capable. */
3208	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3209	return true;
3210	}
3211
3212	return false;
3213	}
3214
3215
3216	/**
3217	* Exports the guest state with appropriate VM-entry controls in the VMCS.
3218	*
3219	* These controls can affect things done on VM-exit; e.g. "load debug controls",
3220	* see Intel spec. 24.8.1 "VM-entry controls".
3221	*
3222	* @returns VBox status code.
3223	* @param pVCpu The cross context virtual CPU structure.
3224	*
3225	* @remarks Requires EFER.
3226	* @remarks No-long-jump zone!!!
3227	*/
3228	static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3229	{
3230	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3231	{
3232	PVM pVM = pVCpu->CTX_SUFF(pVM);
3233	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3234	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3235
3236	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3237	fVal \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3238
3239	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3240	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3241	{
3242	fVal \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3243	Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3244	}
3245	else
3246	Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3247
3248	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3249	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3250	&& hmR0VmxShouldSwapEferMsr(pVCpu))
3251	{
3252	fVal \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3253	Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3254	}
3255
3256	/*
3257	* The following should -not- be set (since we're not in SMM mode):
3258	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3259	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3260	*/
3261
3262	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3263	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3264
3265	if ((fVal & fZap) != fVal)
3266	{
3267	Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3268	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3269	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3270	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3271	}
3272
3273	/* Commit it to the VMCS and update our cache. */
3274	if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3275	{
3276	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3277	AssertRCReturn(rc, rc);
3278	pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3279	}
3280
3281	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3282	}
3283	return VINF_SUCCESS;
3284	}
3285
3286
3287	/**
3288	* Exports the guest state with appropriate VM-exit controls in the VMCS.
3289	*
3290	* @returns VBox status code.
3291	* @param pVCpu The cross context virtual CPU structure.
3292	*
3293	* @remarks Requires EFER.
3294	*/
3295	static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3296	{
3297	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3298	{
3299	PVM pVM = pVCpu->CTX_SUFF(pVM);
3300	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3301	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3302
3303	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3304	fVal \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3305
3306	/*
3307	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3308	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3309	* hmR0VmxExportHostMsrs().
3310	*/
3311	#if HC_ARCH_BITS == 64
3312	fVal \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3313	Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3314	#else
3315	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3316	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3317	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3318	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3319	{
3320	/* The switcher returns to long mode, EFER is managed by the switcher. */
3321	fVal \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3322	Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3323	}
3324	else
3325	Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3326	#endif
3327
3328	/* If the newer VMCS fields for managing EFER exists, use it. */
3329	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3330	&& hmR0VmxShouldSwapEferMsr(pVCpu))
3331	{
3332	fVal \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3333	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3334	Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3335	}
3336
3337	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3338	Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3339
3340	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3341	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3342	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3343
3344	/* Enable saving of the VMX preemption timer value on VM-exit. */
3345	if ( pVM->hm.s.vmx.fUsePreemptTimer
3346	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3347	fVal \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3348
3349	if ((fVal & fZap) != fVal)
3350	{
3351	LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3352	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3353	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3354	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3355	}
3356
3357	/* Commit it to the VMCS and update our cache. */
3358	if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3359	{
3360	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3361	AssertRCReturn(rc, rc);
3362	pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3363	}
3364
3365	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3366	}
3367	return VINF_SUCCESS;
3368	}
3369
3370
3371	/**
3372	* Sets the TPR threshold in the VMCS.
3373	*
3374	* @returns VBox status code.
3375	* @param pVCpu The cross context virtual CPU structure.
3376	* @param u32TprThreshold The TPR threshold (task-priority class only).
3377	*/
3378	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3379	{
3380	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3381	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3382	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3383	}
3384
3385
3386	/**
3387	* Exports the guest APIC TPR state into the VMCS.
3388	*
3389	* @returns VBox status code.
3390	* @param pVCpu The cross context virtual CPU structure.
3391	*
3392	* @remarks No-long-jump zone!!!
3393	*/
3394	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3395	{
3396	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3397	{
3398	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3399
3400	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3401	&& APICIsEnabled(pVCpu))
3402	{
3403	/*
3404	* Setup TPR shadowing.
3405	*/
3406	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407	{
3408	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410	bool fPendingIntr = false;
3411	uint8_t u8Tpr = 0;
3412	uint8_t u8PendingIntr = 0;
3413	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414	AssertRCReturn(rc, rc);
3415
3416	/*
3417	* If there are interrupts pending but masked by the TPR, instruct VT-x to
3418	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3419	* priority of the pending interrupt so we can deliver the interrupt. If there
3420	* are no interrupts pending, set threshold to 0 to not cause any
3421	* TPR-below-threshold VM-exits.
3422	*/
3423	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3424	uint32_t u32TprThreshold = 0;
3425	if (fPendingIntr)
3426	{
3427	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3428	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3429	const uint8_t u8TprPriority = u8Tpr >> 4;
3430	if (u8PendingPriority <= u8TprPriority)
3431	u32TprThreshold = u8PendingPriority;
3432	}
3433
3434	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3435	AssertRCReturn(rc, rc);
3436	}
3437	}
3438	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3439	}
3440	return VINF_SUCCESS;
3441	}
3442
3443
3444	/**
3445	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3446	*
3447	* @returns Guest's interruptibility-state.
3448	* @param pVCpu The cross context virtual CPU structure.
3449	*
3450	* @remarks No-long-jump zone!!!
3451	*/
3452	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3453	{
3454	/*
3455	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3456	*/
3457	uint32_t fIntrState = 0;
3458	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3459	{
3460	/* If inhibition is active, RIP & RFLAGS should've been accessed
3461	(i.e. read previously from the VMCS or from ring-3). */
3462	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3463	#ifdef VBOX_STRICT
3464	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3465	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3466	#endif
3467	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468	{
3469	if (pCtx->eflags.Bits.u1IF)
3470	fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471	else
3472	fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473	}
3474	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475	{
3476	/*
3477	* We can clear the inhibit force flag as even if we go back to the recompiler
3478	* without executing guest code in VT-x, the flag's condition to be cleared is
3479	* met and thus the cleared state is correct.
3480	*/
3481	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3482	}
3483	}
3484
3485	/*
3486	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3487	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3488	* setting this would block host-NMIs and IRET will not clear the blocking.
3489	*
3490	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3491	*/
3492	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3493	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3494	{
3495	fIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3496	}
3497
3498	return fIntrState;
3499	}
3500
3501
3502	/**
3503	* Exports the guest's interruptibility-state into the guest-state area in the
3504	* VMCS.
3505	*
3506	* @returns VBox status code.
3507	* @param pVCpu The cross context virtual CPU structure.
3508	* @param fIntrState The interruptibility-state to set.
3509	*/
3510	static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3511	{
3512	NOREF(pVCpu);
3513	AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3514	Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3515	return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3516	}
3517
3518
3519	/**
3520	* Exports the exception intercepts required for guest execution in the VMCS.
3521	*
3522	* @returns VBox status code.
3523	* @param pVCpu The cross context virtual CPU structure.
3524	*
3525	* @remarks No-long-jump zone!!!
3526	*/
3527	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3528	{
3529	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3530	{
3531	uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3532
3533	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3534	if (pVCpu->hm.s.fGIMTrapXcptUD)
3535	uXcptBitmap \|= RT_BIT(X86_XCPT_UD);
3536	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3537	else
3538	uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3539	#endif
3540
3541	Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3542	Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3543
3544	if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3545	{
3546	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3547	AssertRCReturn(rc, rc);
3548	pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3549	}
3550
3551	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3552	Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3553	}
3554	return VINF_SUCCESS;
3555	}
3556
3557
3558	/**
3559	* Exports the guest's RIP into the guest-state area in the VMCS.
3560	*
3561	* @returns VBox status code.
3562	* @param pVCpu The cross context virtual CPU structure.
3563	*
3564	* @remarks No-long-jump zone!!!
3565	*/
3566	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3567	{
3568	int rc = VINF_SUCCESS;
3569	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3570	{
3571	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3572
3573	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3574	AssertRCReturn(rc, rc);
3575
3576	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3577	Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3578	}
3579	return rc;
3580	}
3581
3582
3583	/**
3584	* Exports the guest's RSP into the guest-state area in the VMCS.
3585	*
3586	* @returns VBox status code.
3587	* @param pVCpu The cross context virtual CPU structure.
3588	*
3589	* @remarks No-long-jump zone!!!
3590	*/
3591	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3592	{
3593	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3594	{
3595	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3596
3597	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3598	AssertRCReturn(rc, rc);
3599
3600	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3601	}
3602	return VINF_SUCCESS;
3603	}
3604
3605
3606	/**
3607	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
3608	*
3609	* @returns VBox status code.
3610	* @param pVCpu The cross context virtual CPU structure.
3611	*
3612	* @remarks No-long-jump zone!!!
3613	*/
3614	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3615	{
3616	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3617	{
3618	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3619
3620	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3621	Let us assert it as such and use 32-bit VMWRITE. */
3622	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3623	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3624	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3625	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3626
3627	/*
3628	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3629	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3630	* can run the real-mode guest code under Virtual 8086 mode.
3631	*/
3632	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3633	{
3634	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3635	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3636	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3637	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3638	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3639	}
3640
3641	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3642	AssertRCReturn(rc, rc);
3643
3644	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3645	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3646	}
3647	return VINF_SUCCESS;
3648	}
3649
3650
3651	/**
3652	* Exports the guest CR0 control register into the guest-state area in the VMCS.
3653	*
3654	* The guest FPU state is always pre-loaded hence we don't need to bother about
3655	* sharing FPU related CR0 bits between the guest and host.
3656	*
3657	* @returns VBox status code.
3658	* @param pVCpu The cross context virtual CPU structure.
3659	*
3660	* @remarks No-long-jump zone!!!
3661	*/
3662	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3663	{
3664	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3665	{
3666	PVM pVM = pVCpu->CTX_SUFF(pVM);
3667	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3668	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3669
3670	uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3671	uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3672
3673	/*
3674	* Setup VT-x's view of the guest CR0.
3675	* Minimize VM-exits due to CR3 changes when we have NestedPaging.
3676	*/
3677	uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3678	if (pVM->hm.s.fNestedPaging)
3679	{
3680	if (CPUMIsGuestPagingEnabled(pVCpu))
3681	{
3682	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3683	uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3684	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3685	}
3686	else
3687	{
3688	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3689	uProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3690	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3691	}
3692
3693	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3694	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3695	uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3696	}
3697	else
3698	{
3699	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3700	u32GuestCr0 \|= X86_CR0_WP;
3701	}
3702
3703	/*
3704	* Guest FPU bits.
3705	*
3706	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3707	* using CR0.TS.
3708	*
3709	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3710	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3711	*/
3712	u32GuestCr0 \|= X86_CR0_NE;
3713
3714	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3715	bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3716
3717	/*
3718	* Update exception intercepts.
3719	*/
3720	uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3721	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3722	{
3723	Assert(PDMVmmDevHeapIsEnabled(pVM));
3724	Assert(pVM->hm.s.vmx.pRealModeTSS);
3725	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3726	}
3727	else
3728	{
3729	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3730	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3731	if (fInterceptMF)
3732	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
3733	}
3734
3735	/* Additional intercepts for debugging, define these yourself explicitly. */
3736	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3737	uXcptBitmap \|= 0
3738	\| RT_BIT(X86_XCPT_BP)
3739	\| RT_BIT(X86_XCPT_DE)
3740	\| RT_BIT(X86_XCPT_NM)
3741	\| RT_BIT(X86_XCPT_TS)
3742	\| RT_BIT(X86_XCPT_UD)
3743	\| RT_BIT(X86_XCPT_NP)
3744	\| RT_BIT(X86_XCPT_SS)
3745	\| RT_BIT(X86_XCPT_GP)
3746	\| RT_BIT(X86_XCPT_PF)
3747	\| RT_BIT(X86_XCPT_MF)
3748	;
3749	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3750	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
3751	#endif
3752	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3753
3754	/*
3755	* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3756	*/
3757	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758	uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3759	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3760	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3761	else
3762	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3763
3764	u32GuestCr0 \|= fSetCr0;
3765	u32GuestCr0 &= fZapCr0;
3766	u32GuestCr0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3767
3768	/*
3769	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3770	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3771	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3772	*/
3773	uint32_t u32Cr0Mask = X86_CR0_PE
3774	\| X86_CR0_NE
3775	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3776	\| X86_CR0_PG
3777	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3778	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3779	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3780
3781	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3782	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3783	* and @bugref{6944}. */
3784	#if 0
3785	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3786	u32Cr0Mask &= ~X86_CR0_PE;
3787	#endif
3788	/*
3789	* Finally, update VMCS fields with the CR0 values and the exception bitmap.
3790	*/
3791	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3792	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3793	if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3794	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3795	if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3796	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3797	if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3798	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3799	AssertRCReturn(rc, rc);
3800
3801	/* Update our caches. */
3802	pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3803	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3804	pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3805
3806	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3807
3808	Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3809	u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3810	}
3811
3812	return VINF_SUCCESS;
3813	}
3814
3815
3816	/**
3817	* Exports the guest control registers (CR3, CR4) into the guest-state area
3818	* in the VMCS.
3819	*
3820	* @returns VBox strict status code.
3821	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3822	* without unrestricted guest access and the VMMDev is not presently
3823	* mapped (e.g. EFI32).
3824	*
3825	* @param pVCpu The cross context virtual CPU structure.
3826	*
3827	* @remarks No-long-jump zone!!!
3828	*/
3829	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3830	{
3831	int rc = VINF_SUCCESS;
3832	PVM pVM = pVCpu->CTX_SUFF(pVM);
3833
3834	/*
3835	* Guest CR2.
3836	* It's always loaded in the assembler code. Nothing to do here.
3837	*/
3838
3839	/*
3840	* Guest CR3.
3841	*/
3842	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3843	{
3844	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3845
3846	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3847	if (pVM->hm.s.fNestedPaging)
3848	{
3849	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3850
3851	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3852	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3853	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3854	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3855
3856	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3857	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3858	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3859
3860	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3861	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3862	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3863	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3864	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3865	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3866	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3867
3868	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3869	AssertRCReturn(rc, rc);
3870
3871	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3872	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3873	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3874	{
3875	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3876	if (CPUMIsGuestInPAEModeEx(pCtx))
3877	{
3878	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3879	AssertRCReturn(rc, rc);
3880	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3881	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3882	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3883	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3884	AssertRCReturn(rc, rc);
3885	}
3886
3887	/*
3888	* The guest's view of its CR3 is unblemished with Nested Paging when the
3889	* guest is using paging or we have unrestricted guest execution to handle
3890	* the guest when it's not using paging.
3891	*/
3892	GCPhysGuestCR3 = pCtx->cr3;
3893	}
3894	else
3895	{
3896	/*
3897	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
3898	* thinks it accesses physical memory directly, we use our identity-mapped
3899	* page table to map guest-linear to guest-physical addresses. EPT takes care
3900	* of translating it to host-physical addresses.
3901	*/
3902	RTGCPHYS GCPhys;
3903	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3904
3905	/* We obtain it here every time as the guest could have relocated this PCI region. */
3906	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3907	if (RT_SUCCESS(rc))
3908	{ /* likely */ }
3909	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3910	{
3911	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3912	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3913	}
3914	else
3915	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3916
3917	GCPhysGuestCR3 = GCPhys;
3918	}
3919
3920	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3921	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3922	AssertRCReturn(rc, rc);
3923	}
3924	else
3925	{
3926	/* Non-nested paging case, just use the hypervisor's CR3. */
3927	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3928
3929	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3930	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3931	AssertRCReturn(rc, rc);
3932	}
3933
3934	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3935	}
3936
3937	/*
3938	* Guest CR4.
3939	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
3940	*/
3941	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3942	{
3943	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3944	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3945	Assert(!RT_HI_U32(pCtx->cr4));
3946
3947	uint32_t u32GuestCr4 = pCtx->cr4;
3948	uint32_t const u32ShadowCr4 = pCtx->cr4;
3949
3950	/*
3951	* Setup VT-x's view of the guest CR4.
3952	*
3953	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3954	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3955	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3956	*
3957	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3958	*/
3959	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3960	{
3961	Assert(pVM->hm.s.vmx.pRealModeTSS);
3962	Assert(PDMVmmDevHeapIsEnabled(pVM));
3963	u32GuestCr4 &= ~X86_CR4_VME;
3964	}
3965
3966	if (pVM->hm.s.fNestedPaging)
3967	{
3968	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3969	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3970	{
3971	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3972	u32GuestCr4 \|= X86_CR4_PSE;
3973	/* Our identity mapping is a 32-bit page directory. */
3974	u32GuestCr4 &= ~X86_CR4_PAE;
3975	}
3976	/* else use guest CR4.*/
3977	}
3978	else
3979	{
3980	/*
3981	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3982	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3983	*/
3984	switch (pVCpu->hm.s.enmShadowMode)
3985	{
3986	case PGMMODE_REAL: /* Real-mode. */
3987	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3988	case PGMMODE_32_BIT: /* 32-bit paging. */
3989	{
3990	u32GuestCr4 &= ~X86_CR4_PAE;
3991	break;
3992	}
3993
3994	case PGMMODE_PAE: /* PAE paging. */
3995	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3996	{
3997	u32GuestCr4 \|= X86_CR4_PAE;
3998	break;
3999	}
4000
4001	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4002	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4003	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4004	break;
4005	#endif
4006	default:
4007	AssertFailed();
4008	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4009	}
4010	}
4011
4012	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4013	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4014	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4015	u32GuestCr4 \|= fSetCr4;
4016	u32GuestCr4 &= fZapCr4;
4017
4018	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4019	that would cause a VM-exit. */
4020	uint32_t u32Cr4Mask = X86_CR4_VME
4021	\| X86_CR4_PAE
4022	\| X86_CR4_PGE
4023	\| X86_CR4_PSE
4024	\| X86_CR4_VMXE;
4025	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4026	u32Cr4Mask \|= X86_CR4_OSXSAVE;
4027	if (pVM->cpum.ro.GuestFeatures.fPcid)
4028	u32Cr4Mask \|= X86_CR4_PCIDE;
4029
4030	/* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4031	into the VMCS and update our cache. */
4032	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4033	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4034	if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4035	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4036	AssertRCReturn(rc, rc);
4037	pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4038
4039	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4040	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4041
4042	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4043
4044	Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4045	fZapCr4));
4046	}
4047	return rc;
4048	}
4049
4050
4051	/**
4052	* Exports the guest debug registers into the guest-state area in the VMCS.
4053	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4054	*
4055	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4056	*
4057	* @returns VBox status code.
4058	* @param pVCpu The cross context virtual CPU structure.
4059	*
4060	* @remarks No-long-jump zone!!!
4061	*/
4062	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4063	{
4064	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4065
4066	#ifdef VBOX_STRICT
4067	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4068	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4069	{
4070	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4071	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
4072	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4073	}
4074	#endif
4075
4076	bool fSteppingDB = false;
4077	bool fInterceptMovDRx = false;
4078	uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4079	if (pVCpu->hm.s.fSingleInstruction)
4080	{
4081	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4082	PVM pVM = pVCpu->CTX_SUFF(pVM);
4083	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4084	{
4085	uProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4086	Assert(fSteppingDB == false);
4087	}
4088	else
4089	{
4090	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
4091	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
4092	pVCpu->hm.s.fClearTrapFlag = true;
4093	fSteppingDB = true;
4094	}
4095	}
4096
4097	uint32_t u32GuestDr7;
4098	if ( fSteppingDB
4099	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4100	{
4101	/*
4102	* Use the combined guest and host DRx values found in the hypervisor register set
4103	* because the debugger has breakpoints active or someone is single stepping on the
4104	* host side without a monitor trap flag.
4105	*
4106	* Note! DBGF expects a clean DR6 state before executing guest code.
4107	*/
4108	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4109	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4110	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4111	{
4112	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4113	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4114	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4115	}
4116	else
4117	#endif
4118	if (!CPUMIsHyperDebugStateActive(pVCpu))
4119	{
4120	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4121	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4122	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4123	}
4124
4125	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4126	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4127	pVCpu->hm.s.fUsingHyperDR7 = true;
4128	fInterceptMovDRx = true;
4129	}
4130	else
4131	{
4132	/*
4133	* If the guest has enabled debug registers, we need to load them prior to
4134	* executing guest code so they'll trigger at the right time.
4135	*/
4136	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
4137	{
4138	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4139	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4140	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4141	{
4142	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4143	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4144	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4145	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4146	}
4147	else
4148	#endif
4149	if (!CPUMIsGuestDebugStateActive(pVCpu))
4150	{
4151	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4152	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4153	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4154	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4155	}
4156	Assert(!fInterceptMovDRx);
4157	}
4158	/*
4159	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4160	* must intercept #DB in order to maintain a correct DR6 guest value, and
4161	* because we need to intercept it to prevent nested #DBs from hanging the
4162	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4163	*/
4164	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4165	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4166	&& !CPUMIsGuestDebugStateActive(pVCpu))
4167	#else
4168	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4169	#endif
4170	{
4171	fInterceptMovDRx = true;
4172	}
4173
4174	/* Update DR7 with the actual guest value. */
4175	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4176	pVCpu->hm.s.fUsingHyperDR7 = false;
4177	}
4178
4179	if (fInterceptMovDRx)
4180	uProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4181	else
4182	uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4183
4184	/*
4185	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4186	* monitor-trap flag and update our cache.
4187	*/
4188	if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4189	{
4190	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4191	AssertRCReturn(rc2, rc2);
4192	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4193	}
4194
4195	/*
4196	* Update guest DR7.
4197	*/
4198	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4199	AssertRCReturn(rc, rc);
4200
4201	return VINF_SUCCESS;
4202	}
4203
4204
4205	#ifdef VBOX_STRICT
4206	/**
4207	* Strict function to validate segment registers.
4208	*
4209	* @param pVCpu The cross context virtual CPU structure.
4210	*
4211	* @remarks Will import guest CR0 on strict builds during validation of
4212	* segments.
4213	*/
4214	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4215	{
4216	/*
4217	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4218	*
4219	* The reason we check for attribute value 0 in this function and not just the unusable bit is
4220	* because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4221	* and doesn't change the guest-context value.
4222	*/
4223	PVM pVM = pVCpu->CTX_SUFF(pVM);
4224	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4225	hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4226	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4227	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4228	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4229	{
4230	/* Protected mode checks */
4231	/* CS */
4232	Assert(pCtx->cs.Attr.n.u1Present);
4233	Assert(!(pCtx->cs.Attr.u & 0xf00));
4234	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4235	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4236	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4237	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4238	\|\| (pCtx->cs.Attr.n.u1Granularity));
4239	/* CS cannot be loaded with NULL in protected mode. */
4240	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4241	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4242	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4243	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4244	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4245	else
4246	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4247	/* SS */
4248	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4249	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4250	if ( !(pCtx->cr0 & X86_CR0_PE)
4251	\|\| pCtx->cs.Attr.n.u4Type == 3)
4252	{
4253	Assert(!pCtx->ss.Attr.n.u2Dpl);
4254	}
4255	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4256	{
4257	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4258	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4259	Assert(pCtx->ss.Attr.n.u1Present);
4260	Assert(!(pCtx->ss.Attr.u & 0xf00));
4261	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4262	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4263	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4264	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4265	\|\| (pCtx->ss.Attr.n.u1Granularity));
4266	}
4267	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4268	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4269	{
4270	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4271	Assert(pCtx->ds.Attr.n.u1Present);
4272	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4273	Assert(!(pCtx->ds.Attr.u & 0xf00));
4274	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4275	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4276	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4277	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4278	\|\| (pCtx->ds.Attr.n.u1Granularity));
4279	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4280	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4281	}
4282	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4283	{
4284	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4285	Assert(pCtx->es.Attr.n.u1Present);
4286	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4287	Assert(!(pCtx->es.Attr.u & 0xf00));
4288	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4289	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4290	\|\| !(pCtx->es.Attr.n.u1Granularity));
4291	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4292	\|\| (pCtx->es.Attr.n.u1Granularity));
4293	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4294	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4295	}
4296	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4297	{
4298	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4299	Assert(pCtx->fs.Attr.n.u1Present);
4300	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4301	Assert(!(pCtx->fs.Attr.u & 0xf00));
4302	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4303	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4304	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4305	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4306	\|\| (pCtx->fs.Attr.n.u1Granularity));
4307	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4308	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4309	}
4310	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4311	{
4312	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4313	Assert(pCtx->gs.Attr.n.u1Present);
4314	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4315	Assert(!(pCtx->gs.Attr.u & 0xf00));
4316	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4317	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4318	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4319	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4320	\|\| (pCtx->gs.Attr.n.u1Granularity));
4321	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4322	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4323	}
4324	/* 64-bit capable CPUs. */
4325	# if HC_ARCH_BITS == 64
4326	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4327	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
4328	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
4329	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
4330	# endif
4331	}
4332	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4333	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4334	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4335	{
4336	/* Real and v86 mode checks. */
4337	/* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4338	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4339	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4340	{
4341	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4342	}
4343	else
4344	{
4345	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4346	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4347	}
4348
4349	/* CS */
4350	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4351	Assert(pCtx->cs.u32Limit == 0xffff);
4352	Assert(u32CSAttr == 0xf3);
4353	/* SS */
4354	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4355	Assert(pCtx->ss.u32Limit == 0xffff);
4356	Assert(u32SSAttr == 0xf3);
4357	/* DS */
4358	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4359	Assert(pCtx->ds.u32Limit == 0xffff);
4360	Assert(u32DSAttr == 0xf3);
4361	/* ES */
4362	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4363	Assert(pCtx->es.u32Limit == 0xffff);
4364	Assert(u32ESAttr == 0xf3);
4365	/* FS */
4366	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4367	Assert(pCtx->fs.u32Limit == 0xffff);
4368	Assert(u32FSAttr == 0xf3);
4369	/* GS */
4370	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4371	Assert(pCtx->gs.u32Limit == 0xffff);
4372	Assert(u32GSAttr == 0xf3);
4373	/* 64-bit capable CPUs. */
4374	# if HC_ARCH_BITS == 64
4375	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4376	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
4377	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
4378	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
4379	# endif
4380	}
4381	}
4382	#endif /* VBOX_STRICT */
4383
4384
4385	/**
4386	* Exports a guest segment register into the guest-state area in the VMCS.
4387	*
4388	* @returns VBox status code.
4389	* @param pVCpu The cross context virtual CPU structure.
4390	* @param idxSel Index of the selector in the VMCS.
4391	* @param idxLimit Index of the segment limit in the VMCS.
4392	* @param idxBase Index of the segment base in the VMCS.
4393	* @param idxAccess Index of the access rights of the segment in the VMCS.
4394	* @param pSelReg Pointer to the segment selector.
4395	*
4396	* @remarks No-long-jump zone!!!
4397	*/
4398	static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4399	PCCPUMSELREG pSelReg)
4400	{
4401	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4402	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4403	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4404	AssertRCReturn(rc, rc);
4405
4406	uint32_t u32Access = pSelReg->Attr.u;
4407	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4408	{
4409	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4410	u32Access = 0xf3;
4411	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4412	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4413	}
4414	else
4415	{
4416	/*
4417	* The way to differentiate between whether this is really a null selector or was just
4418	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
4419	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4420	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4421	* NULL selectors loaded in protected-mode have their attribute as 0.
4422	*/
4423	if (!u32Access)
4424	u32Access = X86DESCATTR_UNUSABLE;
4425	}
4426
4427	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4428	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4429	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4430
4431	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4432	AssertRCReturn(rc, rc);
4433	return rc;
4434	}
4435
4436
4437	/**
4438	* Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4439	* into the guest-state area in the VMCS.
4440	*
4441	* @returns VBox status code.
4442	* @param pVCpu The cross context virtual CPU structure.
4443	*
4444	* @remarks Will import guest CR0 on strict builds during validation of
4445	* segments.
4446	* @remarks No-long-jump zone!!!
4447	*/
4448	static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4449	{
4450	int rc = VERR_INTERNAL_ERROR_5;
4451	PVM pVM = pVCpu->CTX_SUFF(pVM);
4452	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4453
4454	/*
4455	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4456	*/
4457	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4458	{
4459	#ifdef VBOX_WITH_REM
4460	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4461	{
4462	Assert(pVM->hm.s.vmx.pRealModeTSS);
4463	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4464	if ( pVCpu->hm.s.vmx.fWasInRealMode
4465	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4466	{
4467	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4468	in real-mode (e.g. OpenBSD 4.0) */
4469	REMFlushTBs(pVM);
4470	Log4Func(("Switch to protected mode detected!\n"));
4471	pVCpu->hm.s.vmx.fWasInRealMode = false;
4472	}
4473	}
4474	#endif
4475	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4476	{
4477	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4478	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4479	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4480	rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4481	AssertRCReturn(rc, rc);
4482	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4483	}
4484
4485	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4486	{
4487	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4488	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4489	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4490	rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4491	AssertRCReturn(rc, rc);
4492	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4493	}
4494
4495	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4496	{
4497	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4498	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4499	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4500	rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4501	AssertRCReturn(rc, rc);
4502	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4503	}
4504
4505	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4506	{
4507	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4508	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4509	pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4510	rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4511	AssertRCReturn(rc, rc);
4512	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4513	}
4514
4515	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4516	{
4517	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4518	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4519	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4520	rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4521	AssertRCReturn(rc, rc);
4522	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4523	}
4524
4525	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4526	{
4527	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4528	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4529	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4530	rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4531	AssertRCReturn(rc, rc);
4532	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4533	}
4534
4535	#ifdef VBOX_STRICT
4536	hmR0VmxValidateSegmentRegs(pVCpu);
4537	#endif
4538
4539	Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4540	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4541	}
4542
4543	/*
4544	* Guest TR.
4545	*/
4546	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4547	{
4548	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4549
4550	/*
4551	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4552	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4553	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4554	*/
4555	uint16_t u16Sel = 0;
4556	uint32_t u32Limit = 0;
4557	uint64_t u64Base = 0;
4558	uint32_t u32AccessRights = 0;
4559
4560	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4561	{
4562	u16Sel = pCtx->tr.Sel;
4563	u32Limit = pCtx->tr.u32Limit;
4564	u64Base = pCtx->tr.u64Base;
4565	u32AccessRights = pCtx->tr.Attr.u;
4566	}
4567	else
4568	{
4569	Assert(pVM->hm.s.vmx.pRealModeTSS);
4570	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4571
4572	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4573	RTGCPHYS GCPhys;
4574	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4575	AssertRCReturn(rc, rc);
4576
4577	X86DESCATTR DescAttr;
4578	DescAttr.u = 0;
4579	DescAttr.n.u1Present = 1;
4580	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4581
4582	u16Sel = 0;
4583	u32Limit = HM_VTX_TSS_SIZE;
4584	u64Base = GCPhys; /* in real-mode phys = virt. */
4585	u32AccessRights = DescAttr.u;
4586	}
4587
4588	/* Validate. */
4589	Assert(!(u16Sel & RT_BIT(2)));
4590	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4591	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4592	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4593	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4594	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4595	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4596	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4597	Assert( (u32Limit & 0xfff) == 0xfff
4598	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4599	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4600	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4601
4602	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4603	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4604	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4605	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4606	AssertRCReturn(rc, rc);
4607
4608	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4609	Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4610	}
4611
4612	/*
4613	* Guest GDTR.
4614	*/
4615	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4616	{
4617	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4618
4619	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4620	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4621	AssertRCReturn(rc, rc);
4622
4623	/* Validate. */
4624	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4625
4626	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4627	Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4628	}
4629
4630	/*
4631	* Guest LDTR.
4632	*/
4633	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4634	{
4635	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4636
4637	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4638	uint32_t u32Access = 0;
4639	if (!pCtx->ldtr.Attr.u)
4640	u32Access = X86DESCATTR_UNUSABLE;
4641	else
4642	u32Access = pCtx->ldtr.Attr.u;
4643
4644	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4645	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4646	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4647	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4648	AssertRCReturn(rc, rc);
4649
4650	/* Validate. */
4651	if (!(u32Access & X86DESCATTR_UNUSABLE))
4652	{
4653	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4654	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4655	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4656	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4657	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4658	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4659	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4660	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4661	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4662	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4663	}
4664
4665	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4666	Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4667	}
4668
4669	/*
4670	* Guest IDTR.
4671	*/
4672	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4673	{
4674	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4675
4676	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4677	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4678	AssertRCReturn(rc, rc);
4679
4680	/* Validate. */
4681	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4682
4683	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4684	Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4685	}
4686
4687	return VINF_SUCCESS;
4688	}
4689
4690
4691	/**
4692	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4693	* areas.
4694	*
4695	* These MSRs will automatically be loaded to the host CPU on every successful
4696	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4697	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4698	* -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4699	*
4700	* Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4701	*
4702	* @returns VBox status code.
4703	* @param pVCpu The cross context virtual CPU structure.
4704	*
4705	* @remarks No-long-jump zone!!!
4706	*/
4707	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4708	{
4709	AssertPtr(pVCpu);
4710	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4711
4712	/*
4713	* MSRs that we use the auto-load/store MSR area in the VMCS.
4714	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4715	*/
4716	PVM pVM = pVCpu->CTX_SUFF(pVM);
4717	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4718	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4719	{
4720	if (pVM->hm.s.fAllow64BitGuests)
4721	{
4722	#if HC_ARCH_BITS == 32
4723	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
4724
4725	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4726	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4727	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4728	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4729	AssertRCReturn(rc, rc);
4730	# ifdef LOG_ENABLED
4731	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4732	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4733	Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4734	# endif
4735	#endif
4736	}
4737	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738	}
4739
4740	/*
4741	* Guest Sysenter MSRs.
4742	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743	* VM-exits on WRMSRs for these MSRs.
4744	*/
4745	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4746	{
4747	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4748
4749	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4750	{
4751	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4752	AssertRCReturn(rc, rc);
4753	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4754	}
4755
4756	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4757	{
4758	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4759	AssertRCReturn(rc, rc);
4760	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4761	}
4762
4763	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4764	{
4765	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4766	AssertRCReturn(rc, rc);
4767	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4768	}
4769	}
4770
4771	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4772	{
4773	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4774
4775	if (hmR0VmxShouldSwapEferMsr(pVCpu))
4776	{
4777	/*
4778	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4779	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4780	*/
4781	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4782	{
4783	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4784	AssertRCReturn(rc,rc);
4785	Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4786	}
4787	else
4788	{
4789	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4790	NULL /* pfAddedAndUpdated */);
4791	AssertRCReturn(rc, rc);
4792
4793	/* We need to intercept reads too, see @bugref{7386#c16}. */
4794	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4795	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4796	Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4797	pVCpu->hm.s.vmx.cMsrs));
4798	}
4799	}
4800	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4801	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4802	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4803	}
4804
4805	return VINF_SUCCESS;
4806	}
4807
4808
4809	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4810	/**
4811	* Check if guest state allows safe use of 32-bit switcher again.
4812	*
4813	* Segment bases and protected mode structures must be 32-bit addressable
4814	* because the 32-bit switcher will ignore high dword when writing these VMCS
4815	* fields. See @bugref{8432} for details.
4816	*
4817	* @returns true if safe, false if must continue to use the 64-bit switcher.
4818	* @param pCtx Pointer to the guest-CPU context.
4819	*
4820	* @remarks No-long-jump zone!!!
4821	*/
4822	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4823	{
4824	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4825	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4826	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4827	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4828	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4829	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4830	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4831	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4832	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4833	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4834
4835	/* All good, bases are 32-bit. */
4836	return true;
4837	}
4838	#endif
4839
4840
4841	/**
4842	* Selects up the appropriate function to run guest code.
4843	*
4844	* @returns VBox status code.
4845	* @param pVCpu The cross context virtual CPU structure.
4846	*
4847	* @remarks No-long-jump zone!!!
4848	*/
4849	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4850	{
4851	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4852	if (CPUMIsGuestInLongModeEx(pCtx))
4853	{
4854	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4855	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4856	#endif
4857	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4858	#if HC_ARCH_BITS == 32
4859	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4860	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4861	{
4862	#ifdef VBOX_STRICT
4863	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4864	{
4865	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4866	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4867	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4868	AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4869	\| HM_CHANGED_VMX_ENTRY_CTLS
4870	\| HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4871	}
4872	#endif
4873	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4874
4875	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4876	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4877	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4878	Log4Func(("Selected 64-bit switcher\n"));
4879	}
4880	#else
4881	/* 64-bit host. */
4882	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4883	#endif
4884	}
4885	else
4886	{
4887	/* Guest is not in long mode, use the 32-bit handler. */
4888	#if HC_ARCH_BITS == 32
4889	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4890	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4891	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4892	{
4893	# ifdef VBOX_STRICT
4894	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4895	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4896	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4897	AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4898	\| HM_CHANGED_VMX_ENTRY_CTLS
4899	\| HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4900	# endif
4901	}
4902	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4903	/*
4904	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4905	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4906	* switcher flag because now we know the guest is in a sane state where it's safe
4907	* to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4908	* the much faster 32-bit switcher again.
4909	*/
4910	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4911	{
4912	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4913	Log4Func(("Selected 32-bit switcher\n"));
4914	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4915	}
4916	else
4917	{
4918	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4919	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4920	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
4921	{
4922	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4923	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4924	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4925	\| HM_CHANGED_VMX_ENTRY_CTLS
4926	\| HM_CHANGED_VMX_EXIT_CTLS
4927	\| HM_CHANGED_HOST_CONTEXT);
4928	Log4Func(("Selected 32-bit switcher (safe)\n"));
4929	}
4930	}
4931	# else
4932	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4933	# endif
4934	#else
4935	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4936	#endif
4937	}
4938	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4939	return VINF_SUCCESS;
4940	}
4941
4942
4943	/**
4944	* Wrapper for running the guest code in VT-x.
4945	*
4946	* @returns VBox status code, no informational status codes.
4947	* @param pVCpu The cross context virtual CPU structure.
4948	*
4949	* @remarks No-long-jump zone!!!
4950	*/
4951	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4952	{
4953	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4954	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4955	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4956
4957	/*
4958	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4959	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4960	* callee-saved and thus the need for this XMM wrapper.
4961	*
4962	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4963	*/
4964	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4965	/** @todo Add stats for resume vs launch. */
4966	PVM pVM = pVCpu->CTX_SUFF(pVM);
4967	#ifdef VBOX_WITH_KERNEL_USING_XMM
4968	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4969	#else
4970	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4971	#endif
4972	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4973	return rc;
4974	}
4975
4976
4977	/**
4978	* Reports world-switch error and dumps some useful debug info.
4979	*
4980	* @param pVCpu The cross context virtual CPU structure.
4981	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4982	* @param pVmxTransient Pointer to the VMX transient structure (only
4983	* exitReason updated).
4984	*/
4985	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4986	{
4987	Assert(pVCpu);
4988	Assert(pVmxTransient);
4989	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4990
4991	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4992	switch (rcVMRun)
4993	{
4994	case VERR_VMX_INVALID_VMXON_PTR:
4995	AssertFailed();
4996	break;
4997	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4998	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4999	{
5000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5001	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5002	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5003	AssertRC(rc);
5004
5005	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5006	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5007	Cannot do it here as we may have been long preempted. */
5008
5009	#ifdef VBOX_STRICT
5010	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5011	pVmxTransient->uExitReason));
5012	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5013	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5014	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5015	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5016	else
5017	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5018	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5019	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5020
5021	/* VMX control bits. */
5022	uint32_t u32Val;
5023	uint64_t u64Val;
5024	RTHCUINTREG uHCReg;
5025	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5026	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5027	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5028	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5029	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5030	{
5031	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5032	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5033	}
5034	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5035	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5036	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5037	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5038	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5039	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5040	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5041	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5042	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5043	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5044	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5045	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5046	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5047	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5048	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5049	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5050	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5051	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5052	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5053	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5054	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5055	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5056	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5057	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5058	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5059	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5060	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5061	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5062	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5063	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5064	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5065	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5066	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5067	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5068	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5069	{
5070	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5071	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5072	}
5073
5074	/* Guest bits. */
5075	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5076	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5077	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5078	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5079	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5080	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5081	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5082	{
5083	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5084	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5085	}
5086
5087	/* Host bits. */
5088	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5089	Log4(("Host CR0 %#RHr\n", uHCReg));
5090	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5091	Log4(("Host CR3 %#RHr\n", uHCReg));
5092	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5093	Log4(("Host CR4 %#RHr\n", uHCReg));
5094
5095	RTGDTR HostGdtr;
5096	PCX86DESCHC pDesc;
5097	ASMGetGDTR(&HostGdtr);
5098	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5099	Log4(("Host CS %#08x\n", u32Val));
5100	if (u32Val < HostGdtr.cbGdt)
5101	{
5102	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5103	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5104	}
5105
5106	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5107	Log4(("Host DS %#08x\n", u32Val));
5108	if (u32Val < HostGdtr.cbGdt)
5109	{
5110	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5111	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5112	}
5113
5114	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5115	Log4(("Host ES %#08x\n", u32Val));
5116	if (u32Val < HostGdtr.cbGdt)
5117	{
5118	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5119	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5120	}
5121
5122	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5123	Log4(("Host FS %#08x\n", u32Val));
5124	if (u32Val < HostGdtr.cbGdt)
5125	{
5126	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5127	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5128	}
5129
5130	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5131	Log4(("Host GS %#08x\n", u32Val));
5132	if (u32Val < HostGdtr.cbGdt)
5133	{
5134	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5135	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5136	}
5137
5138	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5139	Log4(("Host SS %#08x\n", u32Val));
5140	if (u32Val < HostGdtr.cbGdt)
5141	{
5142	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5143	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5144	}
5145
5146	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5147	Log4(("Host TR %#08x\n", u32Val));
5148	if (u32Val < HostGdtr.cbGdt)
5149	{
5150	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5151	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5152	}
5153
5154	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5155	Log4(("Host TR Base %#RHv\n", uHCReg));
5156	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5157	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5158	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5159	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5160	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5161	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5162	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5163	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5164	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5165	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5166	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5167	Log4(("Host RSP %#RHv\n", uHCReg));
5168	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5169	Log4(("Host RIP %#RHv\n", uHCReg));
5170	# if HC_ARCH_BITS == 64
5171	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5172	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5173	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5174	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5175	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5176	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5177	# endif
5178	#endif /* VBOX_STRICT */
5179	break;
5180	}
5181
5182	default:
5183	/* Impossible */
5184	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5185	break;
5186	}
5187	}
5188
5189
5190	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5191	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5192	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5193	#endif
5194	#ifdef VBOX_STRICT
5195	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5196	{
5197	switch (idxField)
5198	{
5199	case VMX_VMCS_GUEST_RIP:
5200	case VMX_VMCS_GUEST_RSP:
5201	case VMX_VMCS_GUEST_SYSENTER_EIP:
5202	case VMX_VMCS_GUEST_SYSENTER_ESP:
5203	case VMX_VMCS_GUEST_GDTR_BASE:
5204	case VMX_VMCS_GUEST_IDTR_BASE:
5205	case VMX_VMCS_GUEST_CS_BASE:
5206	case VMX_VMCS_GUEST_DS_BASE:
5207	case VMX_VMCS_GUEST_ES_BASE:
5208	case VMX_VMCS_GUEST_FS_BASE:
5209	case VMX_VMCS_GUEST_GS_BASE:
5210	case VMX_VMCS_GUEST_SS_BASE:
5211	case VMX_VMCS_GUEST_LDTR_BASE:
5212	case VMX_VMCS_GUEST_TR_BASE:
5213	case VMX_VMCS_GUEST_CR3:
5214	return true;
5215	}
5216	return false;
5217	}
5218
5219	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5220	{
5221	switch (idxField)
5222	{
5223	/* Read-only fields. */
5224	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5225	return true;
5226	}
5227	/* Remaining readable fields should also be writable. */
5228	return hmR0VmxIsValidWriteField(idxField);
5229	}
5230	#endif /* VBOX_STRICT */
5231
5232
5233	/**
5234	* Executes the specified handler in 64-bit mode.
5235	*
5236	* @returns VBox status code (no informational status codes).
5237	* @param pVCpu The cross context virtual CPU structure.
5238	* @param enmOp The operation to perform.
5239	* @param cParams Number of parameters.
5240	* @param paParam Array of 32-bit parameters.
5241	*/
5242	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5243	{
5244	PVM pVM = pVCpu->CTX_SUFF(pVM);
5245	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5246	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5247	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5248	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5249
5250	#ifdef VBOX_STRICT
5251	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5252	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5253
5254	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5255	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5256	#endif
5257
5258	/* Disable interrupts. */
5259	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5260
5261	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5262	RTCPUID idHostCpu = RTMpCpuId();
5263	CPUMR0SetLApic(pVCpu, idHostCpu);
5264	#endif
5265
5266	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5267	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5268
5269	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5270	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5271	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5272
5273	/* Leave VMX Root Mode. */
5274	VMXDisable();
5275
5276	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5277
5278	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5279	CPUMSetHyperEIP(pVCpu, enmOp);
5280	for (int i = (int)cParams - 1; i >= 0; i--)
5281	CPUMPushHyper(pVCpu, paParam[i]);
5282
5283	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5284
5285	/* Call the switcher. */
5286	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5287	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5288
5289	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5290	/* Make sure the VMX instructions don't cause #UD faults. */
5291	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5292
5293	/* Re-enter VMX Root Mode */
5294	int rc2 = VMXEnable(HCPhysCpuPage);
5295	if (RT_FAILURE(rc2))
5296	{
5297	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5298	ASMSetFlags(fOldEFlags);
5299	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5300	return rc2;
5301	}
5302
5303	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5304	AssertRC(rc2);
5305	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5306	Assert(!(ASMGetFlags() & X86_EFL_IF));
5307	ASMSetFlags(fOldEFlags);
5308	return rc;
5309	}
5310
5311
5312	/**
5313	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5314	* supporting 64-bit guests.
5315	*
5316	* @returns VBox status code.
5317	* @param fResume Whether to VMLAUNCH or VMRESUME.
5318	* @param pCtx Pointer to the guest-CPU context.
5319	* @param pCache Pointer to the VMCS cache.
5320	* @param pVM The cross context VM structure.
5321	* @param pVCpu The cross context virtual CPU structure.
5322	*/
5323	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5324	{
5325	NOREF(fResume);
5326
5327	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5328	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5329
5330	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5331	pCache->uPos = 1;
5332	pCache->interPD = PGMGetInterPaeCR3(pVM);
5333	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5334	#endif
5335
5336	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5337	pCache->TestIn.HCPhysCpuPage = 0;
5338	pCache->TestIn.HCPhysVmcs = 0;
5339	pCache->TestIn.pCache = 0;
5340	pCache->TestOut.HCPhysVmcs = 0;
5341	pCache->TestOut.pCache = 0;
5342	pCache->TestOut.pCtx = 0;
5343	pCache->TestOut.eflags = 0;
5344	#else
5345	NOREF(pCache);
5346	#endif
5347
5348	uint32_t aParam[10];
5349	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5350	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5351	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5352	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5353	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5354	aParam[5] = 0;
5355	aParam[6] = VM_RC_ADDR(pVM, pVM);
5356	aParam[7] = 0;
5357	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5358	aParam[9] = 0;
5359
5360	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5361	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5362	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5363	#endif
5364	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5365
5366	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5367	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5368	Assert(pCtx->dr[4] == 10);
5369	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5370	#endif
5371
5372	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5373	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5374	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5375	pVCpu->hm.s.vmx.HCPhysVmcs));
5376	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5377	pCache->TestOut.HCPhysVmcs));
5378	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5379	pCache->TestOut.pCache));
5380	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5381	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5382	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5383	pCache->TestOut.pCtx));
5384	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5385	#endif
5386	NOREF(pCtx);
5387	return rc;
5388	}
5389
5390
5391	/**
5392	* Initialize the VMCS-Read cache.
5393	*
5394	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5395	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5396	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5397	* (those that have a 32-bit FULL & HIGH part).
5398	*
5399	* @returns VBox status code.
5400	* @param pVCpu The cross context virtual CPU structure.
5401	*/
5402	static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5403	{
5404	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5405	do { \
5406	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5407	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5408	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5409	++cReadFields; \
5410	} while (0)
5411
5412	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5413	uint32_t cReadFields = 0;
5414
5415	/*
5416	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5417	* and serve to indicate exceptions to the rules.
5418	*/
5419
5420	/* Guest-natural selector base fields. */
5421	#if 0
5422	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5423	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5424	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5425	#endif
5426	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5427	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5428	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5429	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5430	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5431	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5432	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5433	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5434	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5435	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5436	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5437	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5438	#if 0
5439	/* Unused natural width guest-state fields. */
5440	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5441	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5442	#endif
5443	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5444	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5445
5446	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5447	these 64-bit fields (using "FULL" and "HIGH" fields). */
5448	#if 0
5449	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5450	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5451	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5452	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5453	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5454	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5455	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5456	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5457	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5458	#endif
5459
5460	/* Natural width guest-state fields. */
5461	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5462	#if 0
5463	/* Currently unused field. */
5464	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5465	#endif
5466
5467	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5468	{
5469	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5470	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5471	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5472	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5473	}
5474	else
5475	{
5476	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5477	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5478	}
5479
5480	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5481	return VINF_SUCCESS;
5482	}
5483
5484
5485	/**
5486	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5487	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5488	* darwin, running 64-bit guests).
5489	*
5490	* @returns VBox status code.
5491	* @param pVCpu The cross context virtual CPU structure.
5492	* @param idxField The VMCS field encoding.
5493	* @param u64Val 16, 32 or 64-bit value.
5494	*/
5495	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5496	{
5497	int rc;
5498	switch (idxField)
5499	{
5500	/*
5501	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5502	*/
5503	/* 64-bit Control fields. */
5504	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5505	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5506	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5507	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5508	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5509	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5510	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5511	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5512	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5513	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5514	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5515	case VMX_VMCS64_CTRL_EPTP_FULL:
5516	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5517	/* 64-bit Guest-state fields. */
5518	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5519	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5520	case VMX_VMCS64_GUEST_PAT_FULL:
5521	case VMX_VMCS64_GUEST_EFER_FULL:
5522	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5523	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5524	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5525	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5526	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5527	/* 64-bit Host-state fields. */
5528	case VMX_VMCS64_HOST_PAT_FULL:
5529	case VMX_VMCS64_HOST_EFER_FULL:
5530	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5531	{
5532	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5533	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5534	break;
5535	}
5536
5537	/*
5538	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5539	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5540	*/
5541	/* Natural-width Guest-state fields. */
5542	case VMX_VMCS_GUEST_CR3:
5543	case VMX_VMCS_GUEST_ES_BASE:
5544	case VMX_VMCS_GUEST_CS_BASE:
5545	case VMX_VMCS_GUEST_SS_BASE:
5546	case VMX_VMCS_GUEST_DS_BASE:
5547	case VMX_VMCS_GUEST_FS_BASE:
5548	case VMX_VMCS_GUEST_GS_BASE:
5549	case VMX_VMCS_GUEST_LDTR_BASE:
5550	case VMX_VMCS_GUEST_TR_BASE:
5551	case VMX_VMCS_GUEST_GDTR_BASE:
5552	case VMX_VMCS_GUEST_IDTR_BASE:
5553	case VMX_VMCS_GUEST_RSP:
5554	case VMX_VMCS_GUEST_RIP:
5555	case VMX_VMCS_GUEST_SYSENTER_ESP:
5556	case VMX_VMCS_GUEST_SYSENTER_EIP:
5557	{
5558	if (!(RT_HI_U32(u64Val)))
5559	{
5560	/* If this field is 64-bit, VT-x will zero out the top bits. */
5561	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5562	}
5563	else
5564	{
5565	/* Assert that only the 32->64 switcher case should ever come here. */
5566	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5567	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5568	}
5569	break;
5570	}
5571
5572	default:
5573	{
5574	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5575	rc = VERR_INVALID_PARAMETER;
5576	break;
5577	}
5578	}
5579	AssertRCReturn(rc, rc);
5580	return rc;
5581	}
5582
5583
5584	/**
5585	* Queue up a VMWRITE by using the VMCS write cache.
5586	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5587	*
5588	* @param pVCpu The cross context virtual CPU structure.
5589	* @param idxField The VMCS field encoding.
5590	* @param u64Val 16, 32 or 64-bit value.
5591	*/
5592	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5593	{
5594	AssertPtr(pVCpu);
5595	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5596
5597	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5598	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5599
5600	/* Make sure there are no duplicates. */
5601	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5602	{
5603	if (pCache->Write.aField[i] == idxField)
5604	{
5605	pCache->Write.aFieldVal[i] = u64Val;
5606	return VINF_SUCCESS;
5607	}
5608	}
5609
5610	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5611	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5612	pCache->Write.cValidEntries++;
5613	return VINF_SUCCESS;
5614	}
5615	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5616
5617
5618	/**
5619	* Sets up the usage of TSC-offsetting and updates the VMCS.
5620	*
5621	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5622	* VMX preemption timer.
5623	*
5624	* @returns VBox status code.
5625	* @param pVCpu The cross context virtual CPU structure.
5626	*
5627	* @remarks No-long-jump zone!!!
5628	*/
5629	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5630	{
5631	bool fOffsettedTsc;
5632	bool fParavirtTsc;
5633	PVM pVM = pVCpu->CTX_SUFF(pVM);
5634	uint64_t uTscOffset;
5635	if (pVM->hm.s.vmx.fUsePreemptTimer)
5636	{
5637	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5638
5639	/* Make sure the returned values have sane upper and lower boundaries. */
5640	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5641	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5642	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5643	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5644
5645	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5646	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5647	AssertRC(rc);
5648	}
5649	else
5650	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5651
5652	/** @todo later optimize this to be done elsewhere and not before every
5653	* VM-entry. */
5654	if (fParavirtTsc)
5655	{
5656	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5657	information before every VM-entry, hence disable it for performance sake. */
5658	#if 0
5659	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5660	AssertRC(rc);
5661	#endif
5662	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5663	}
5664
5665	uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5666	if ( fOffsettedTsc
5667	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5668	{
5669	if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5670	{
5671	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5672	AssertRC(rc);
5673	pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5674	}
5675
5676	if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5677	{
5678	uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5679	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5680	AssertRC(rc);
5681	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5682	}
5683	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5684	}
5685	else
5686	{
5687	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5688	if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5689	{
5690	uProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5691	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5692	AssertRC(rc);
5693	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5694	}
5695	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5696	}
5697	}
5698
5699
5700	/**
5701	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5702	* VM-exit interruption info type.
5703	*
5704	* @returns The IEM exception flags.
5705	* @param uVector The event vector.
5706	* @param uVmxVectorType The VMX event type.
5707	*
5708	* @remarks This function currently only constructs flags required for
5709	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5710	* and CR2 aspects of an exception are not included).
5711	*/
5712	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5713	{
5714	uint32_t fIemXcptFlags;
5715	switch (uVmxVectorType)
5716	{
5717	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5718	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5719	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5720	break;
5721
5722	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5723	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5724	break;
5725
5726	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5727	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5728	break;
5729
5730	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5731	{
5732	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5733	if (uVector == X86_XCPT_BP)
5734	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5735	else if (uVector == X86_XCPT_OF)
5736	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5737	else
5738	{
5739	fIemXcptFlags = 0;
5740	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5741	}
5742	break;
5743	}
5744
5745	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5746	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5747	break;
5748
5749	default:
5750	fIemXcptFlags = 0;
5751	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5752	break;
5753	}
5754	return fIemXcptFlags;
5755	}
5756
5757
5758	/**
5759	* Sets an event as a pending event to be injected into the guest.
5760	*
5761	* @param pVCpu The cross context virtual CPU structure.
5762	* @param u32IntInfo The VM-entry interruption-information field.
5763	* @param cbInstr The VM-entry instruction length in bytes (for software
5764	* interrupts, exceptions and privileged software
5765	* exceptions).
5766	* @param u32ErrCode The VM-entry exception error code.
5767	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5768	* page-fault.
5769	*
5770	* @remarks Statistics counter assumes this is a guest event being injected or
5771	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5772	* always incremented.
5773	*/
5774	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5775	RTGCUINTPTR GCPtrFaultAddress)
5776	{
5777	Assert(!pVCpu->hm.s.Event.fPending);
5778	pVCpu->hm.s.Event.fPending = true;
5779	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5780	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5781	pVCpu->hm.s.Event.cbInstr = cbInstr;
5782	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5783	}
5784
5785
5786	/**
5787	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5788	*
5789	* @param pVCpu The cross context virtual CPU structure.
5790	*/
5791	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5792	{
5793	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5794	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5795	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5796	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5797	}
5798
5799
5800	/**
5801	* Handle a condition that occurred while delivering an event through the guest
5802	* IDT.
5803	*
5804	* @returns Strict VBox status code (i.e. informational status codes too).
5805	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5806	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5807	* to continue execution of the guest which will delivery the \#DF.
5808	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5809	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5810	*
5811	* @param pVCpu The cross context virtual CPU structure.
5812	* @param pVmxTransient Pointer to the VMX transient structure.
5813	*
5814	* @remarks No-long-jump zone!!!
5815	*/
5816	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5817	{
5818	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5819
5820	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5821	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5822	AssertRCReturn(rc2, rc2);
5823
5824	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5825	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5826	{
5827	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5828	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5829
5830	/*
5831	* If the event was a software interrupt (generated with INT n) or a software exception
5832	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5833	* can handle the VM-exit and continue guest execution which will re-execute the
5834	* instruction rather than re-injecting the exception, as that can cause premature
5835	* trips to ring-3 before injection and involve TRPM which currently has no way of
5836	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5837	* the problem).
5838	*/
5839	IEMXCPTRAISE enmRaise;
5840	IEMXCPTRAISEINFO fRaiseInfo;
5841	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5842	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5843	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5844	{
5845	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5846	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5847	}
5848	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5849	{
5850	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5851	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5852	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5853	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5854	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5855	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5856	uExitVectorType), VERR_VMX_IPE_5);
5857
5858	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5859
5860	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5861	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5862	{
5863	pVmxTransient->fVectoringPF = true;
5864	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5865	}
5866	}
5867	else
5868	{
5869	/*
5870	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5871	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5872	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5873	*/
5874	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5875	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5876	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5877	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5878	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5879	}
5880
5881	/*
5882	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5883	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5884	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5885	* subsequent VM-entry would fail.
5886	*
5887	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5888	*/
5889	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5890	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5891	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5892	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5893	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5894	{
5895	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5896	}
5897
5898	switch (enmRaise)
5899	{
5900	case IEMXCPTRAISE_CURRENT_XCPT:
5901	{
5902	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5903	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5904	Assert(rcStrict == VINF_SUCCESS);
5905	break;
5906	}
5907
5908	case IEMXCPTRAISE_PREV_EVENT:
5909	{
5910	uint32_t u32ErrCode;
5911	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5912	{
5913	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5914	AssertRCReturn(rc2, rc2);
5915	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5916	}
5917	else
5918	u32ErrCode = 0;
5919
5920	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5921	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5922	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5923	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
5924
5925	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5926	pVCpu->hm.s.Event.u32ErrCode));
5927	Assert(rcStrict == VINF_SUCCESS);
5928	break;
5929	}
5930
5931	case IEMXCPTRAISE_REEXEC_INSTR:
5932	Assert(rcStrict == VINF_SUCCESS);
5933	break;
5934
5935	case IEMXCPTRAISE_DOUBLE_FAULT:
5936	{
5937	/*
5938	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5939	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5940	*/
5941	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5942	{
5943	pVmxTransient->fVectoringDoublePF = true;
5944	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5945	pVCpu->cpum.GstCtx.cr2));
5946	rcStrict = VINF_SUCCESS;
5947	}
5948	else
5949	{
5950	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5951	hmR0VmxSetPendingXcptDF(pVCpu);
5952	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5953	uIdtVector, uExitVector));
5954	rcStrict = VINF_HM_DOUBLE_FAULT;
5955	}
5956	break;
5957	}
5958
5959	case IEMXCPTRAISE_TRIPLE_FAULT:
5960	{
5961	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5962	rcStrict = VINF_EM_RESET;
5963	break;
5964	}
5965
5966	case IEMXCPTRAISE_CPU_HANG:
5967	{
5968	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5969	rcStrict = VERR_EM_GUEST_CPU_HANG;
5970	break;
5971	}
5972
5973	default:
5974	{
5975	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5976	rcStrict = VERR_VMX_IPE_2;
5977	break;
5978	}
5979	}
5980	}
5981	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5982	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5983	&& uExitVector != X86_XCPT_DF
5984	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5985	{
5986	/*
5987	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5988	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5989	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5990	*/
5991	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5992	{
5993	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5994	VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5995	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5996	}
5997	}
5998
5999	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6000	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6001	return rcStrict;
6002	}
6003
6004
6005	/**
6006	* Imports a guest segment register from the current VMCS into
6007	* the guest-CPU context.
6008	*
6009	* @returns VBox status code.
6010	* @param pVCpu The cross context virtual CPU structure.
6011	* @param idxSel Index of the selector in the VMCS.
6012	* @param idxLimit Index of the segment limit in the VMCS.
6013	* @param idxBase Index of the segment base in the VMCS.
6014	* @param idxAccess Index of the access rights of the segment in the VMCS.
6015	* @param pSelReg Pointer to the segment selector.
6016	*
6017	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
6018	* do not log!
6019	*
6020	* @remarks Never call this function directly!!! Use the
6021	* HMVMX_IMPORT_SREG() macro as that takes care
6022	* of whether to read from the VMCS cache or not.
6023	*/
6024	static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6025	PCPUMSELREG pSelReg)
6026	{
6027	NOREF(pVCpu);
6028
6029	uint32_t u32Sel;
6030	uint32_t u32Limit;
6031	uint32_t u32Attr;
6032	uint64_t u64Base;
6033	int rc = VMXReadVmcs32(idxSel, &u32Sel);
6034	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
6035	rc \|= VMXReadVmcs32(idxAccess, &u32Attr);
6036	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6037	AssertRCReturn(rc, rc);
6038
6039	pSelReg->Sel = (uint16_t)u32Sel;
6040	pSelReg->ValidSel = (uint16_t)u32Sel;
6041	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6042	pSelReg->u32Limit = u32Limit;
6043	pSelReg->u64Base = u64Base;
6044	pSelReg->Attr.u = u32Attr;
6045
6046	/*
6047	* If VT-x marks the segment as unusable, most other bits remain undefined:
6048	* - For CS the L, D and G bits have meaning.
6049	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6050	* - For the remaining data segments no bits are defined.
6051	*
6052	* The present bit and the unusable bit has been observed to be set at the
6053	* same time (the selector was supposed to be invalid as we started executing
6054	* a V8086 interrupt in ring-0).
6055	*
6056	* What should be important for the rest of the VBox code, is that the P bit is
6057	* cleared. Some of the other VBox code recognizes the unusable bit, but
6058	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6059	* safe side here, we'll strip off P and other bits we don't care about. If
6060	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6061	*
6062	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6063	*/
6064	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6065	{
6066	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6067
6068	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6069	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6070	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6071	#ifdef VBOX_STRICT
6072	VMMRZCallRing3Disable(pVCpu);
6073	Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6074	# ifdef DEBUG_bird
6075	AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6076	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6077	idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6078	# endif
6079	VMMRZCallRing3Enable(pVCpu);
6080	#endif
6081	}
6082	return VINF_SUCCESS;
6083	}
6084
6085
6086	/**
6087	* Imports the guest RIP from the VMCS back into the guest-CPU context.
6088	*
6089	* @returns VBox status code.
6090	* @param pVCpu The cross context virtual CPU structure.
6091	*
6092	* @remarks Called with interrupts and/or preemption disabled, should not assert!
6093	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6094	* instead!!!
6095	*/
6096	DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6097	{
6098	uint64_t u64Val;
6099	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6100	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6101	{
6102	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6103	if (RT_SUCCESS(rc))
6104	{
6105	pCtx->rip = u64Val;
6106	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6107	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6108	}
6109	return rc;
6110	}
6111	return VINF_SUCCESS;
6112	}
6113
6114
6115	/**
6116	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6117	*
6118	* @returns VBox status code.
6119	* @param pVCpu The cross context virtual CPU structure.
6120	*
6121	* @remarks Called with interrupts and/or preemption disabled, should not assert!
6122	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6123	* instead!!!
6124	*/
6125	DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6126	{
6127	uint32_t u32Val;
6128	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6129	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6130	{
6131	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6132	if (RT_SUCCESS(rc))
6133	{
6134	pCtx->eflags.u32 = u32Val;
6135
6136	/* Restore eflags for real-on-v86-mode hack. */
6137	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6138	{
6139	pCtx->eflags.Bits.u1VM = 0;
6140	pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6141	}
6142	}
6143	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6144	return rc;
6145	}
6146	return VINF_SUCCESS;
6147	}
6148
6149
6150	/**
6151	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6152	* context.
6153	*
6154	* @returns VBox status code.
6155	* @param pVCpu The cross context virtual CPU structure.
6156	*
6157	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
6158	* do not log!
6159	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6160	* instead!!!
6161	*/
6162	DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6163	{
6164	uint32_t u32Val;
6165	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6166	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6167	if (RT_SUCCESS(rc))
6168	{
6169	/*
6170	* We additionally have a requirement to import RIP, RFLAGS depending on whether we
6171	* might need them in hmR0VmxEvaluatePendingEvent().
6172	*/
6173	if (!u32Val)
6174	{
6175	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6176	{
6177	rc = hmR0VmxImportGuestRip(pVCpu);
6178	rc \|= hmR0VmxImportGuestRFlags(pVCpu);
6179	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6180	}
6181
6182	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6183	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6184	}
6185	else
6186	{
6187	rc = hmR0VmxImportGuestRip(pVCpu);
6188	rc \|= hmR0VmxImportGuestRFlags(pVCpu);
6189
6190	if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6191	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6192	{
6193	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6194	}
6195	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6196	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6197
6198	if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6199	{
6200	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6201	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6202	}
6203	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6204	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6205	}
6206	}
6207	return rc;
6208	}
6209
6210
6211	/**
6212	* Worker for VMXR0ImportStateOnDemand.
6213	*
6214	* @returns VBox status code.
6215	* @param pVCpu The cross context virtual CPU structure.
6216	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6217	*/
6218	static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6219	{
6220	#define VMXLOCAL_BREAK_RC(a_rc) \
6221	if (RT_FAILURE(a_rc)) \
6222	break
6223
6224	int rc = VINF_SUCCESS;
6225	PVM pVM = pVCpu->CTX_SUFF(pVM);
6226	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6227	uint64_t u64Val;
6228	uint32_t u32Val;
6229
6230	Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6231	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6232
6233	/*
6234	* We disable interrupts to make the updating of the state and in particular
6235	* the fExtrn modification atomic wrt to preemption hooks.
6236	*/
6237	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6238
6239	fWhat &= pCtx->fExtrn;
6240	if (fWhat)
6241	{
6242	do
6243	{
6244	if (fWhat & CPUMCTX_EXTRN_RIP)
6245	{
6246	rc = hmR0VmxImportGuestRip(pVCpu);
6247	VMXLOCAL_BREAK_RC(rc);
6248	}
6249
6250	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6251	{
6252	rc = hmR0VmxImportGuestRFlags(pVCpu);
6253	VMXLOCAL_BREAK_RC(rc);
6254	}
6255
6256	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6257	{
6258	rc = hmR0VmxImportGuestIntrState(pVCpu);
6259	VMXLOCAL_BREAK_RC(rc);
6260	}
6261
6262	if (fWhat & CPUMCTX_EXTRN_RSP)
6263	{
6264	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6265	VMXLOCAL_BREAK_RC(rc);
6266	pCtx->rsp = u64Val;
6267	}
6268
6269	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6270	{
6271	if (fWhat & CPUMCTX_EXTRN_CS)
6272	{
6273	rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6274	rc \|= hmR0VmxImportGuestRip(pVCpu);
6275	VMXLOCAL_BREAK_RC(rc);
6276	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6277	pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6278	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6279	}
6280	if (fWhat & CPUMCTX_EXTRN_SS)
6281	{
6282	rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6283	VMXLOCAL_BREAK_RC(rc);
6284	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6285	pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6286	}
6287	if (fWhat & CPUMCTX_EXTRN_DS)
6288	{
6289	rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6290	VMXLOCAL_BREAK_RC(rc);
6291	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6292	pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6293	}
6294	if (fWhat & CPUMCTX_EXTRN_ES)
6295	{
6296	rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6297	VMXLOCAL_BREAK_RC(rc);
6298	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6299	pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6300	}
6301	if (fWhat & CPUMCTX_EXTRN_FS)
6302	{
6303	rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6304	VMXLOCAL_BREAK_RC(rc);
6305	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6306	pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6307	}
6308	if (fWhat & CPUMCTX_EXTRN_GS)
6309	{
6310	rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6311	VMXLOCAL_BREAK_RC(rc);
6312	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6313	pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6314	}
6315	}
6316
6317	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6318	{
6319	if (fWhat & CPUMCTX_EXTRN_LDTR)
6320	{
6321	rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6322	VMXLOCAL_BREAK_RC(rc);
6323	}
6324
6325	if (fWhat & CPUMCTX_EXTRN_GDTR)
6326	{
6327	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6328	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6329	VMXLOCAL_BREAK_RC(rc);
6330	pCtx->gdtr.pGdt = u64Val;
6331	pCtx->gdtr.cbGdt = u32Val;
6332	}
6333
6334	/* Guest IDTR. */
6335	if (fWhat & CPUMCTX_EXTRN_IDTR)
6336	{
6337	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6338	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6339	VMXLOCAL_BREAK_RC(rc);
6340	pCtx->idtr.pIdt = u64Val;
6341	pCtx->idtr.cbIdt = u32Val;
6342	}
6343
6344	/* Guest TR. */
6345	if (fWhat & CPUMCTX_EXTRN_TR)
6346	{
6347	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6348	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6349	{
6350	rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6351	VMXLOCAL_BREAK_RC(rc);
6352	}
6353	}
6354	}
6355
6356	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6357	{
6358	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6359	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6360	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6361	pCtx->SysEnter.cs = u32Val;
6362	VMXLOCAL_BREAK_RC(rc);
6363	}
6364
6365	#if HC_ARCH_BITS == 64
6366	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6367	{
6368	if ( pVM->hm.s.fAllow64BitGuests
6369	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6370	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6371	}
6372
6373	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6374	{
6375	if ( pVM->hm.s.fAllow64BitGuests
6376	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6377	{
6378	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6379	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6380	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6381	}
6382	}
6383	#endif
6384
6385	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
6386	#if HC_ARCH_BITS == 32
6387	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
6388	#endif
6389	)
6390	{
6391	PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6392	uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6393	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6394	{
6395	switch (pMsr->u32Msr)
6396	{
6397	#if HC_ARCH_BITS == 32
6398	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6399	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6400	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6401	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6402	#endif
6403	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6404	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6405	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6406	default:
6407	{
6408	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6409	ASMSetFlags(fEFlags);
6410	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6411	cMsrs));
6412	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6413	}
6414	}
6415	}
6416	}
6417
6418	if (fWhat & CPUMCTX_EXTRN_DR7)
6419	{
6420	if (!pVCpu->hm.s.fUsingHyperDR7)
6421	{
6422	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6423	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6424	VMXLOCAL_BREAK_RC(rc);
6425	pCtx->dr[7] = u32Val;
6426	}
6427	}
6428
6429	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6430	{
6431	uint32_t u32Shadow;
6432	if (fWhat & CPUMCTX_EXTRN_CR0)
6433	{
6434	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6435	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6436	VMXLOCAL_BREAK_RC(rc);
6437	u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6438	\| (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6439	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6440	CPUMSetGuestCR0(pVCpu, u32Val);
6441	VMMRZCallRing3Enable(pVCpu);
6442	}
6443
6444	if (fWhat & CPUMCTX_EXTRN_CR4)
6445	{
6446	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6447	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6448	VMXLOCAL_BREAK_RC(rc);
6449	u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6450	\| (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6451	CPUMSetGuestCR4(pVCpu, u32Val);
6452	}
6453
6454	if (fWhat & CPUMCTX_EXTRN_CR3)
6455	{
6456	/* CR0.PG bit changes are always intercepted, so it's up to date. */
6457	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6458	\|\| ( pVM->hm.s.fNestedPaging
6459	&& CPUMIsGuestPagingEnabledEx(pCtx)))
6460	{
6461	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6462	if (pCtx->cr3 != u64Val)
6463	{
6464	CPUMSetGuestCR3(pVCpu, u64Val);
6465	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6466	}
6467
6468	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6469	Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6470	if (CPUMIsGuestInPAEModeEx(pCtx))
6471	{
6472	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6473	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6474	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6475	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6476	VMXLOCAL_BREAK_RC(rc);
6477	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6478	}
6479	}
6480	}
6481	}
6482	} while (0);
6483
6484	if (RT_SUCCESS(rc))
6485	{
6486	/* Update fExtrn. */
6487	pCtx->fExtrn &= ~fWhat;
6488
6489	/* If everything has been imported, clear the HM keeper bit. */
6490	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6491	{
6492	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6493	Assert(!pCtx->fExtrn);
6494	}
6495	}
6496	}
6497	else
6498	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6499
6500	ASMSetFlags(fEFlags);
6501
6502	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6503
6504	/*
6505	* Honor any pending CR3 updates.
6506	*
6507	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6508	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6509	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6510	*
6511	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6512	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6513	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6514	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6515	*
6516	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6517	*/
6518	if (VMMRZCallRing3IsEnabled(pVCpu))
6519	{
6520	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6521	{
6522	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6523	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6524	}
6525
6526	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6527	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6528
6529	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6530	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6531	}
6532
6533	return VINF_SUCCESS;
6534	#undef VMXLOCAL_BREAK_RC
6535	}
6536
6537
6538	/**
6539	* Saves the guest state from the VMCS into the guest-CPU context.
6540	*
6541	* @returns VBox status code.
6542	* @param pVCpu The cross context virtual CPU structure.
6543	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6544	*/
6545	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6546	{
6547	return hmR0VmxImportGuestState(pVCpu, fWhat);
6548	}
6549
6550
6551	/**
6552	* Check per-VM and per-VCPU force flag actions that require us to go back to
6553	* ring-3 for one reason or another.
6554	*
6555	* @returns Strict VBox status code (i.e. informational status codes too)
6556	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6557	* ring-3.
6558	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6559	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6560	* interrupts)
6561	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6562	* all EMTs to be in ring-3.
6563	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6564	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6565	* to the EM loop.
6566	*
6567	* @param pVCpu The cross context virtual CPU structure.
6568	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
6569	*/
6570	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6571	{
6572	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6573
6574	/*
6575	* Anything pending? Should be more likely than not if we're doing a good job.
6576	*/
6577	PVM pVM = pVCpu->CTX_SUFF(pVM);
6578	if ( !fStepping
6579	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6580	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6581	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6582	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6583	return VINF_SUCCESS;
6584
6585	/* Pending PGM C3 sync. */
6586	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6587	{
6588	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6589	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
6590	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6591	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6592	if (rcStrict2 != VINF_SUCCESS)
6593	{
6594	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6595	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6596	return rcStrict2;
6597	}
6598	}
6599
6600	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6601	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6602	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6603	{
6604	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6605	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6606	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6607	return rc2;
6608	}
6609
6610	/* Pending VM request packets, such as hardware interrupts. */
6611	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6612	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6613	{
6614	Log4Func(("Pending VM request forcing us back to ring-3\n"));
6615	return VINF_EM_PENDING_REQUEST;
6616	}
6617
6618	/* Pending PGM pool flushes. */
6619	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6620	{
6621	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6622	return VINF_PGM_POOL_FLUSH_PENDING;
6623	}
6624
6625	/* Pending DMA requests. */
6626	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6627	{
6628	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6629	return VINF_EM_RAW_TO_R3;
6630	}
6631
6632	return VINF_SUCCESS;
6633	}
6634
6635
6636	/**
6637	* Converts any TRPM trap into a pending HM event. This is typically used when
6638	* entering from ring-3 (not longjmp returns).
6639	*
6640	* @param pVCpu The cross context virtual CPU structure.
6641	*/
6642	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6643	{
6644	Assert(TRPMHasTrap(pVCpu));
6645	Assert(!pVCpu->hm.s.Event.fPending);
6646
6647	uint8_t uVector;
6648	TRPMEVENT enmTrpmEvent;
6649	RTGCUINT uErrCode;
6650	RTGCUINTPTR GCPtrFaultAddress;
6651	uint8_t cbInstr;
6652
6653	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6654	AssertRC(rc);
6655
6656	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6657	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6658	if (enmTrpmEvent == TRPM_TRAP)
6659	{
6660	switch (uVector)
6661	{
6662	case X86_XCPT_NMI:
6663	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6664	break;
6665
6666	case X86_XCPT_BP:
6667	case X86_XCPT_OF:
6668	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6669	break;
6670
6671	case X86_XCPT_PF:
6672	case X86_XCPT_DF:
6673	case X86_XCPT_TS:
6674	case X86_XCPT_NP:
6675	case X86_XCPT_SS:
6676	case X86_XCPT_GP:
6677	case X86_XCPT_AC:
6678	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6679	RT_FALL_THRU();
6680	default:
6681	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6682	break;
6683	}
6684	}
6685	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6686	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6687	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6688	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6689	else
6690	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6691
6692	rc = TRPMResetTrap(pVCpu);
6693	AssertRC(rc);
6694	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6695	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6696
6697	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6698	}
6699
6700
6701	/**
6702	* Converts the pending HM event into a TRPM trap.
6703	*
6704	* @param pVCpu The cross context virtual CPU structure.
6705	*/
6706	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6707	{
6708	Assert(pVCpu->hm.s.Event.fPending);
6709
6710	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6711	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6712	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6713	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6714
6715	/* If a trap was already pending, we did something wrong! */
6716	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6717
6718	TRPMEVENT enmTrapType;
6719	switch (uVectorType)
6720	{
6721	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6722	enmTrapType = TRPM_HARDWARE_INT;
6723	break;
6724
6725	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6726	enmTrapType = TRPM_SOFTWARE_INT;
6727	break;
6728
6729	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6730	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6731	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6732	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6733	enmTrapType = TRPM_TRAP;
6734	break;
6735
6736	default:
6737	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6738	enmTrapType = TRPM_32BIT_HACK;
6739	break;
6740	}
6741
6742	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6743
6744	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6745	AssertRC(rc);
6746
6747	if (fErrorCodeValid)
6748	TRPMSetErrorCode(pVCpu, uErrorCode);
6749
6750	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6751	&& uVector == X86_XCPT_PF)
6752	{
6753	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6754	}
6755	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6756	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6757	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6758	{
6759	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6760	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6761	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6762	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6763	}
6764
6765	/* Clear any pending events from the VMCS. */
6766	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6767	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6768
6769	/* We're now done converting the pending event. */
6770	pVCpu->hm.s.Event.fPending = false;
6771	}
6772
6773
6774	/**
6775	* Does the necessary state syncing before returning to ring-3 for any reason
6776	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6777	*
6778	* @returns VBox status code.
6779	* @param pVCpu The cross context virtual CPU structure.
6780	* @param fImportState Whether to import the guest state from the VMCS back
6781	* to the guest-CPU context.
6782	*
6783	* @remarks No-long-jmp zone!!!
6784	*/
6785	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6786	{
6787	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6788	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6789
6790	RTCPUID idCpu = RTMpCpuId();
6791	Log4Func(("HostCpuId=%u\n", idCpu));
6792
6793	/*
6794	* !!! IMPORTANT !!!
6795	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6796	*/
6797
6798	/* Save the guest state if necessary. */
6799	if (fImportState)
6800	{
6801	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6802	AssertRCReturn(rc, rc);
6803	}
6804
6805	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6806	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6807	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6808
6809	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6810	#ifdef VBOX_STRICT
6811	if (CPUMIsHyperDebugStateActive(pVCpu))
6812	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6813	#endif
6814	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6815	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6816	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6817
6818	#if HC_ARCH_BITS == 64
6819	/* Restore host-state bits that VT-x only restores partially. */
6820	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6821	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6822	{
6823	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6824	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6825	}
6826	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6827	#endif
6828
6829	/* Restore the lazy host MSRs as we're leaving VT-x context. */
6830	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6831	{
6832	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6833	if (!fImportState)
6834	{
6835	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
6836	AssertRCReturn(rc, rc);
6837	}
6838	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6839	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6840	}
6841	else
6842	pVCpu->hm.s.vmx.fLazyMsrs = 0;
6843
6844	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6845	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6846
6847	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6848	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6849	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6850	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6851	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6852	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6853	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6854	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6855	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6856
6857	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6858
6859	/** @todo This partially defeats the purpose of having preemption hooks.
6860	* The problem is, deregistering the hooks should be moved to a place that
6861	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6862	* context.
6863	*/
6864	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6865	{
6866	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6867	AssertRCReturn(rc, rc);
6868
6869	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6870	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6871	}
6872	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6873	NOREF(idCpu);
6874
6875	return VINF_SUCCESS;
6876	}
6877
6878
6879	/**
6880	* Leaves the VT-x session.
6881	*
6882	* @returns VBox status code.
6883	* @param pVCpu The cross context virtual CPU structure.
6884	*
6885	* @remarks No-long-jmp zone!!!
6886	*/
6887	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
6888	{
6889	HM_DISABLE_PREEMPT(pVCpu);
6890	HMVMX_ASSERT_CPU_SAFE(pVCpu);
6891	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6892	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6893
6894	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6895	and done this from the VMXR0ThreadCtxCallback(). */
6896	if (!pVCpu->hm.s.fLeaveDone)
6897	{
6898	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6899	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6900	pVCpu->hm.s.fLeaveDone = true;
6901	}
6902	Assert(!pVCpu->cpum.GstCtx.fExtrn);
6903
6904	/*
6905	* !!! IMPORTANT !!!
6906	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6907	*/
6908
6909	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6910	/** @todo Deregistering here means we need to VMCLEAR always
6911	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6912	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6913	VMMR0ThreadCtxHookDisable(pVCpu);
6914
6915	/* Leave HM context. This takes care of local init (term). */
6916	int rc = HMR0LeaveCpu(pVCpu);
6917
6918	HM_RESTORE_PREEMPT();
6919	return rc;
6920	}
6921
6922
6923	/**
6924	* Does the necessary state syncing before doing a longjmp to ring-3.
6925	*
6926	* @returns VBox status code.
6927	* @param pVCpu The cross context virtual CPU structure.
6928	*
6929	* @remarks No-long-jmp zone!!!
6930	*/
6931	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
6932	{
6933	return hmR0VmxLeaveSession(pVCpu);
6934	}
6935
6936
6937	/**
6938	* Take necessary actions before going back to ring-3.
6939	*
6940	* An action requires us to go back to ring-3. This function does the necessary
6941	* steps before we can safely return to ring-3. This is not the same as longjmps
6942	* to ring-3, this is voluntary and prepares the guest so it may continue
6943	* executing outside HM (recompiler/IEM).
6944	*
6945	* @returns VBox status code.
6946	* @param pVCpu The cross context virtual CPU structure.
6947	* @param rcExit The reason for exiting to ring-3. Can be
6948	* VINF_VMM_UNKNOWN_RING3_CALL.
6949	*/
6950	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
6951	{
6952	Assert(pVCpu);
6953	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6954
6955	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6956	{
6957	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6958	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6959	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6960	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6961	}
6962
6963	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6964	VMMRZCallRing3Disable(pVCpu);
6965	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6966
6967	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6968	if (pVCpu->hm.s.Event.fPending)
6969	{
6970	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6971	Assert(!pVCpu->hm.s.Event.fPending);
6972	}
6973
6974	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6975	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6976
6977	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6978	and if we're injecting an event we should have a TRPM trap pending. */
6979	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6980	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6981	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6982	#endif
6983
6984	/* Save guest state and restore host state bits. */
6985	int rc = hmR0VmxLeaveSession(pVCpu);
6986	AssertRCReturn(rc, rc);
6987	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6988	/* Thread-context hooks are unregistered at this point!!! */
6989
6990	/* Sync recompiler state. */
6991	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6992	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6993	\| CPUM_CHANGED_LDTR
6994	\| CPUM_CHANGED_GDTR
6995	\| CPUM_CHANGED_IDTR
6996	\| CPUM_CHANGED_TR
6997	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6998	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
6999	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7000	{
7001	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7002	}
7003
7004	Assert(!pVCpu->hm.s.fClearTrapFlag);
7005
7006	/* Update the exit-to-ring 3 reason. */
7007	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7008
7009	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7010	if (rcExit != VINF_EM_RAW_INTERRUPT)
7011	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7012
7013	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7014
7015	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7016	VMMRZCallRing3RemoveNotification(pVCpu);
7017	VMMRZCallRing3Enable(pVCpu);
7018
7019	return rc;
7020	}
7021
7022
7023	/**
7024	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7025	* longjump to ring-3 and possibly get preempted.
7026	*
7027	* @returns VBox status code.
7028	* @param pVCpu The cross context virtual CPU structure.
7029	* @param enmOperation The operation causing the ring-3 longjump.
7030	* @param pvUser User argument, currently unused, NULL.
7031	*/
7032	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7033	{
7034	RT_NOREF(pvUser);
7035	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7036	{
7037	/*
7038	* !!! IMPORTANT !!!
7039	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7040	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7041	*/
7042	VMMRZCallRing3RemoveNotification(pVCpu);
7043	VMMRZCallRing3Disable(pVCpu);
7044	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7045	RTThreadPreemptDisable(&PreemptState);
7046
7047	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7048	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7049	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7050
7051	#if HC_ARCH_BITS == 64
7052	/* Restore host-state bits that VT-x only restores partially. */
7053	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7054	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7055	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7056	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7057	#endif
7058
7059	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7060	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7061	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7062
7063	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7064	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7065	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7066	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7067	{
7068	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7069	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7070	}
7071
7072	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7073	VMMR0ThreadCtxHookDisable(pVCpu);
7074	HMR0LeaveCpu(pVCpu);
7075	RTThreadPreemptRestore(&PreemptState);
7076	return VINF_SUCCESS;
7077	}
7078
7079	Assert(pVCpu);
7080	Assert(pvUser);
7081	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7082	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7083
7084	VMMRZCallRing3Disable(pVCpu);
7085	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7086
7087	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7088
7089	int rc = hmR0VmxLongJmpToRing3(pVCpu);
7090	AssertRCReturn(rc, rc);
7091
7092	VMMRZCallRing3Enable(pVCpu);
7093	return VINF_SUCCESS;
7094	}
7095
7096
7097	/**
7098	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7099	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7100	*
7101	* @param pVCpu The cross context virtual CPU structure.
7102	*/
7103	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7104	{
7105	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7106	{
7107	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7108	{
7109	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7110	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7111	AssertRC(rc);
7112	Log4Func(("Setup interrupt-window exiting\n"));
7113	}
7114	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7115	}
7116
7117
7118	/**
7119	* Clears the interrupt-window exiting control in the VMCS.
7120	*
7121	* @param pVCpu The cross context virtual CPU structure.
7122	*/
7123	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7124	{
7125	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7126	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7127	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7128	AssertRC(rc);
7129	Log4Func(("Cleared interrupt-window exiting\n"));
7130	}
7131
7132
7133	/**
7134	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7135	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7136	*
7137	* @param pVCpu The cross context virtual CPU structure.
7138	*/
7139	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7140	{
7141	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7142	{
7143	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7144	{
7145	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7146	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7147	AssertRC(rc);
7148	Log4Func(("Setup NMI-window exiting\n"));
7149	}
7150	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7151	}
7152
7153
7154	/**
7155	* Clears the NMI-window exiting control in the VMCS.
7156	*
7157	* @param pVCpu The cross context virtual CPU structure.
7158	*/
7159	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7160	{
7161	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7162	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7163	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7164	AssertRC(rc);
7165	Log4Func(("Cleared NMI-window exiting\n"));
7166	}
7167
7168
7169	/**
7170	* Evaluates the event to be delivered to the guest and sets it as the pending
7171	* event.
7172	*
7173	* @returns The VT-x guest-interruptibility state.
7174	* @param pVCpu The cross context virtual CPU structure.
7175	*/
7176	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7177	{
7178	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7179	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7180	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7181	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7182	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7183	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7184
7185	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7186	Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7187	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7188	Assert(!TRPMHasTrap(pVCpu));
7189
7190	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7191	APICUpdatePendingInterrupts(pVCpu);
7192
7193	/*
7194	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7195	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7196	*/
7197	/** @todo SMI. SMIs take priority over NMIs. */
7198	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7199	{
7200	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7201	if ( !pVCpu->hm.s.Event.fPending
7202	&& !fBlockNmi
7203	&& !fBlockSti
7204	&& !fBlockMovSS)
7205	{
7206	Log4Func(("Pending NMI\n"));
7207	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7208	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7209
7210	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7211	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7212	}
7213	else
7214	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7215	}
7216	/*
7217	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7218	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7219	*/
7220	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7221	&& !pVCpu->hm.s.fSingleInstruction)
7222	{
7223	Assert(!DBGFIsStepping(pVCpu));
7224	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7225	AssertRCReturn(rc, 0);
7226	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7227	if ( !pVCpu->hm.s.Event.fPending
7228	&& !fBlockInt
7229	&& !fBlockSti
7230	&& !fBlockMovSS)
7231	{
7232	uint8_t u8Interrupt;
7233	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7234	if (RT_SUCCESS(rc))
7235	{
7236	Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7237	uint32_t u32IntInfo = u8Interrupt
7238	\| VMX_EXIT_INTERRUPTION_INFO_VALID
7239	\| (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7240
7241	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7242	}
7243	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7244	{
7245	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7246	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7247	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7248
7249	/*
7250	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7251	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7252	* need to re-set this force-flag here.
7253	*/
7254	}
7255	else
7256	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7257	}
7258	else
7259	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7260	}
7261
7262	return fIntrState;
7263	}
7264
7265
7266	/**
7267	* Sets a pending-debug exception to be delivered to the guest if the guest is
7268	* single-stepping in the VMCS.
7269	*
7270	* @param pVCpu The cross context virtual CPU structure.
7271	*/
7272	DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7273	{
7274	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7275	RT_NOREF(pVCpu);
7276	return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7277	}
7278
7279
7280	/**
7281	* Injects any pending events into the guest if the guest is in a state to
7282	* receive them.
7283	*
7284	* @returns Strict VBox status code (i.e. informational status codes too).
7285	* @param pVCpu The cross context virtual CPU structure.
7286	* @param fIntrState The VT-x guest-interruptibility state.
7287	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7288	* return VINF_EM_DBG_STEPPED if the event was
7289	* dispatched directly.
7290	*/
7291	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7292	{
7293	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7294	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7295
7296	bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7297	bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7298
7299	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7300	Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7301	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7302	Assert(!TRPMHasTrap(pVCpu));
7303
7304	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7305	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7306	if (pVCpu->hm.s.Event.fPending)
7307	{
7308	/*
7309	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7310	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7311	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7312	*
7313	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7314	*/
7315	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7316	#ifdef VBOX_STRICT
7317	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7318	{
7319	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7320	Assert(!fBlockInt);
7321	Assert(!fBlockSti);
7322	Assert(!fBlockMovSS);
7323	}
7324	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7325	{
7326	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7327	Assert(!fBlockSti);
7328	Assert(!fBlockMovSS);
7329	Assert(!fBlockNmi);
7330	}
7331	#endif
7332	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7333	uIntType));
7334	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7335	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7336	&fIntrState);
7337	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7338
7339	/* Update the interruptibility-state as it could have been changed by
7340	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7341	fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7342	fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7343
7344	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7345	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7346	else
7347	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7348	}
7349
7350	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7351	if ( fBlockSti
7352	\|\| fBlockMovSS)
7353	{
7354	if (!pVCpu->hm.s.fSingleInstruction)
7355	{
7356	/*
7357	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7358	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7359	* See Intel spec. 27.3.4 "Saving Non-Register State".
7360	*/
7361	Assert(!DBGFIsStepping(pVCpu));
7362	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7363	AssertRCReturn(rc, rc);
7364	if (pCtx->eflags.Bits.u1TF)
7365	{
7366	int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7367	AssertRCReturn(rc2, rc2);
7368	}
7369	}
7370	else if (pCtx->eflags.Bits.u1TF)
7371	{
7372	/*
7373	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7374	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7375	*/
7376	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7377	fIntrState = 0;
7378	}
7379	}
7380
7381	/*
7382	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7383	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7384	*/
7385	int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7386	AssertRCReturn(rc3, rc3);
7387
7388	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7389	NOREF(fBlockMovSS); NOREF(fBlockSti);
7390	return rcStrict;
7391	}
7392
7393
7394	/**
7395	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7396	*
7397	* @param pVCpu The cross context virtual CPU structure.
7398	*/
7399	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7400	{
7401	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7402	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7403	}
7404
7405
7406	/**
7407	* Injects a double-fault (\#DF) exception into the VM.
7408	*
7409	* @returns Strict VBox status code (i.e. informational status codes too).
7410	* @param pVCpu The cross context virtual CPU structure.
7411	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7412	* and should return VINF_EM_DBG_STEPPED if the event
7413	* is injected directly (register modified by us, not
7414	* by hardware on VM-entry).
7415	* @param pfIntrState Pointer to the current guest interruptibility-state.
7416	* This interruptibility-state will be updated if
7417	* necessary. This cannot not be NULL.
7418	*/
7419	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7420	{
7421	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7422	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7423	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7424	return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */, fStepping,
7425	pfIntrState);
7426	}
7427
7428
7429	/**
7430	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7431	*
7432	* @param pVCpu The cross context virtual CPU structure.
7433	*/
7434	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7435	{
7436	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7437	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7438	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7439	}
7440
7441
7442	/**
7443	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7444	*
7445	* @param pVCpu The cross context virtual CPU structure.
7446	* @param cbInstr The value of RIP that is to be pushed on the guest
7447	* stack.
7448	*/
7449	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, uint32_t cbInstr)
7450	{
7451	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7452	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7453	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7454	}
7455
7456
7457	/**
7458	* Injects a general-protection (\#GP) fault into the VM.
7459	*
7460	* @returns Strict VBox status code (i.e. informational status codes too).
7461	* @param pVCpu The cross context virtual CPU structure.
7462	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7463	* mode, i.e. in real-mode it's not valid).
7464	* @param u32ErrorCode The error code associated with the \#GP.
7465	* @param fStepping Whether we're running in
7466	* hmR0VmxRunGuestCodeStep() and should return
7467	* VINF_EM_DBG_STEPPED if the event is injected
7468	* directly (register modified by us, not by
7469	* hardware on VM-entry).
7470	* @param pfIntrState Pointer to the current guest interruptibility-state.
7471	* This interruptibility-state will be updated if
7472	* necessary. This cannot not be NULL.
7473	*/
7474	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7475	uint32_t *pfIntrState)
7476	{
7477	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7478	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7479	if (fErrorCodeValid)
7480	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7481	return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */, fStepping,
7482	pfIntrState);
7483	}
7484
7485
7486	/**
7487	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7488	*
7489	* @param pVCpu The cross context virtual CPU structure.
7490	* @param uVector The software interrupt vector number.
7491	* @param cbInstr The value of RIP that is to be pushed on the guest
7492	* stack.
7493	*/
7494	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, uint16_t uVector, uint32_t cbInstr)
7495	{
7496	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7497	if ( uVector == X86_XCPT_BP
7498	\|\| uVector == X86_XCPT_OF)
7499	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7500	else
7501	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7502	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7503	}
7504
7505
7506	/**
7507	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7508	* stack.
7509	*
7510	* @returns Strict VBox status code (i.e. informational status codes too).
7511	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7512	* @param pVCpu The cross context virtual CPU structure.
7513	* @param uValue The value to push to the guest stack.
7514	*/
7515	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7516	{
7517	/*
7518	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7519	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7520	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7521	*/
7522	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7523	if (pCtx->sp == 1)
7524	return VINF_EM_RESET;
7525	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7526	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7527	AssertRC(rc);
7528	return rc;
7529	}
7530
7531
7532	/**
7533	* Injects an event into the guest upon VM-entry by updating the relevant fields
7534	* in the VM-entry area in the VMCS.
7535	*
7536	* @returns Strict VBox status code (i.e. informational status codes too).
7537	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7538	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7539	*
7540	* @param pVCpu The cross context virtual CPU structure.
7541	* @param u64IntInfo The VM-entry interruption-information field.
7542	* @param cbInstr The VM-entry instruction length in bytes (for
7543	* software interrupts, exceptions and privileged
7544	* software exceptions).
7545	* @param u32ErrCode The VM-entry exception error code.
7546	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7547	* @param pfIntrState Pointer to the current guest interruptibility-state.
7548	* This interruptibility-state will be updated if
7549	* necessary. This cannot not be NULL.
7550	* @param fStepping Whether we're running in
7551	* hmR0VmxRunGuestCodeStep() and should return
7552	* VINF_EM_DBG_STEPPED if the event is injected
7553	* directly (register modified by us, not by
7554	* hardware on VM-entry).
7555	*/
7556	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7557	RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7558	{
7559	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7560	AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7561	Assert(pfIntrState);
7562
7563	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7564	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7565	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7566	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7567
7568	#ifdef VBOX_STRICT
7569	/*
7570	* Validate the error-code-valid bit for hardware exceptions.
7571	* No error codes for exceptions in real-mode.
7572	*
7573	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7574	*/
7575	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7576	&& !CPUMIsGuestInRealModeEx(pCtx))
7577	{
7578	switch (uVector)
7579	{
7580	case X86_XCPT_PF:
7581	case X86_XCPT_DF:
7582	case X86_XCPT_TS:
7583	case X86_XCPT_NP:
7584	case X86_XCPT_SS:
7585	case X86_XCPT_GP:
7586	case X86_XCPT_AC:
7587	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7588	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7589	RT_FALL_THRU();
7590	default:
7591	break;
7592	}
7593	}
7594	#endif
7595
7596	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7597	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7598	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7599
7600	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7601
7602	/*
7603	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
7604	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7605	* interrupt handler in the (real-mode) guest.
7606	*
7607	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7608	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7609	*/
7610	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7611	{
7612	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7613	{
7614	/*
7615	* For unrestricted execution enabled CPUs running real-mode guests, we must not
7616	* set the deliver-error-code bit.
7617	*
7618	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7619	*/
7620	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7621	}
7622	else
7623	{
7624	PVM pVM = pVCpu->CTX_SUFF(pVM);
7625	Assert(PDMVmmDevHeapIsEnabled(pVM));
7626	Assert(pVM->hm.s.vmx.pRealModeTSS);
7627
7628	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7629	int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK \| CPUMCTX_EXTRN_RIP
7630	\| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
7631	AssertRCReturn(rc2, rc2);
7632
7633	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7634	size_t const cbIdtEntry = sizeof(X86IDTR16);
7635	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7636	{
7637	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7638	if (uVector == X86_XCPT_DF)
7639	return VINF_EM_RESET;
7640
7641	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7642	if (uVector == X86_XCPT_GP)
7643	return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7644
7645	/*
7646	* If we're injecting an event with no valid IDT entry, inject a #GP.
7647	* No error codes for exceptions in real-mode.
7648	*
7649	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7650	*/
7651	return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid /, 0 / u32ErrCode */, fStepping, pfIntrState);
7652	}
7653
7654	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7655	uint16_t uGuestIp = pCtx->ip;
7656	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7657	{
7658	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7659	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7660	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7661	}
7662	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7663	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7664
7665	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7666	X86IDTR16 IdtEntry;
7667	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7668	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7669	AssertRCReturn(rc2, rc2);
7670
7671	/* Construct the stack frame for the interrupt/exception handler. */
7672	VBOXSTRICTRC rcStrict;
7673	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7674	if (rcStrict == VINF_SUCCESS)
7675	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7676	if (rcStrict == VINF_SUCCESS)
7677	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7678
7679	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7680	if (rcStrict == VINF_SUCCESS)
7681	{
7682	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7683	pCtx->rip = IdtEntry.offSel;
7684	pCtx->cs.Sel = IdtEntry.uSel;
7685	pCtx->cs.ValidSel = IdtEntry.uSel;
7686	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7687	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7688	&& uVector == X86_XCPT_PF)
7689	pCtx->cr2 = GCPtrFaultAddress;
7690
7691	/* If any other guest-state bits are changed here, make sure to update
7692	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7693	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
7694	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
7695	\| HM_CHANGED_GUEST_RSP);
7696
7697	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7698	if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7699	{
7700	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7701	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7702	Log4Func(("Clearing inhibition due to STI\n"));
7703	*pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7704	}
7705	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7706	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7707
7708	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7709	it, if we are returning to ring-3 before executing guest code. */
7710	pVCpu->hm.s.Event.fPending = false;
7711
7712	/* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7713	if (fStepping)
7714	rcStrict = VINF_EM_DBG_STEPPED;
7715	}
7716	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7717	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7718	return rcStrict;
7719	}
7720	}
7721
7722	/* Validate. */
7723	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7724	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7725	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7726
7727	/* Inject. */
7728	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7729	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7730	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7731	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7732	AssertRCReturn(rc, rc);
7733
7734	/* Update CR2. */
7735	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7736	&& uVector == X86_XCPT_PF)
7737	pCtx->cr2 = GCPtrFaultAddress;
7738
7739	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7740
7741	return VINF_SUCCESS;
7742	}
7743
7744
7745	/**
7746	* Clears the interrupt-window exiting control in the VMCS and if necessary
7747	* clears the current event in the VMCS as well.
7748	*
7749	* @returns VBox status code.
7750	* @param pVCpu The cross context virtual CPU structure.
7751	*
7752	* @remarks Use this function only to clear events that have not yet been
7753	* delivered to the guest but are injected in the VMCS!
7754	* @remarks No-long-jump zone!!!
7755	*/
7756	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7757	{
7758	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7759	{
7760	hmR0VmxClearIntWindowExitVmcs(pVCpu);
7761	Log4Func(("Cleared interrupt widow\n"));
7762	}
7763
7764	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7765	{
7766	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7767	Log4Func(("Cleared interrupt widow\n"));
7768	}
7769	}
7770
7771
7772	/**
7773	* Enters the VT-x session.
7774	*
7775	* @returns VBox status code.
7776	* @param pVCpu The cross context virtual CPU structure.
7777	* @param pHostCpu Pointer to the global CPU info struct.
7778	*/
7779	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7780	{
7781	AssertPtr(pVCpu);
7782	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7783	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7784	RT_NOREF(pHostCpu);
7785
7786	LogFlowFunc(("pVCpu=%p\n", pVCpu));
7787	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7788	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7789
7790	#ifdef VBOX_STRICT
7791	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7792	RTCCUINTREG uHostCR4 = ASMGetCR4();
7793	if (!(uHostCR4 & X86_CR4_VMXE))
7794	{
7795	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7796	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7797	}
7798	#endif
7799
7800	/*
7801	* Load the VCPU's VMCS as the current (and active) one.
7802	*/
7803	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7804	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7805	if (RT_FAILURE(rc))
7806	return rc;
7807
7808	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7809	pVCpu->hm.s.fLeaveDone = false;
7810	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7811
7812	return VINF_SUCCESS;
7813	}
7814
7815
7816	/**
7817	* The thread-context callback (only on platforms which support it).
7818	*
7819	* @param enmEvent The thread-context event.
7820	* @param pVCpu The cross context virtual CPU structure.
7821	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7822	* @thread EMT(pVCpu)
7823	*/
7824	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7825	{
7826	NOREF(fGlobalInit);
7827
7828	switch (enmEvent)
7829	{
7830	case RTTHREADCTXEVENT_OUT:
7831	{
7832	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7833	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7834	VMCPU_ASSERT_EMT(pVCpu);
7835
7836	/* No longjmps (logger flushes, locks) in this fragile context. */
7837	VMMRZCallRing3Disable(pVCpu);
7838	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7839
7840	/*
7841	* Restore host-state (FPU, debug etc.)
7842	*/
7843	if (!pVCpu->hm.s.fLeaveDone)
7844	{
7845	/*
7846	* Do -not- import the guest-state here as we might already be in the middle of importing
7847	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7848	*/
7849	hmR0VmxLeave(pVCpu, false /* fImportState */);
7850	pVCpu->hm.s.fLeaveDone = true;
7851	}
7852
7853	/* Leave HM context, takes care of local init (term). */
7854	int rc = HMR0LeaveCpu(pVCpu);
7855	AssertRC(rc); NOREF(rc);
7856
7857	/* Restore longjmp state. */
7858	VMMRZCallRing3Enable(pVCpu);
7859	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7860	break;
7861	}
7862
7863	case RTTHREADCTXEVENT_IN:
7864	{
7865	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7866	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7867	VMCPU_ASSERT_EMT(pVCpu);
7868
7869	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7870	VMMRZCallRing3Disable(pVCpu);
7871	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7872
7873	/* Initialize the bare minimum state required for HM. This takes care of
7874	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7875	int rc = hmR0EnterCpu(pVCpu);
7876	AssertRC(rc);
7877	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7878	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7879
7880	/* Load the active VMCS as the current one. */
7881	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7882	{
7883	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7884	AssertRC(rc); NOREF(rc);
7885	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7886	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7887	}
7888	pVCpu->hm.s.fLeaveDone = false;
7889
7890	/* Restore longjmp state. */
7891	VMMRZCallRing3Enable(pVCpu);
7892	break;
7893	}
7894
7895	default:
7896	break;
7897	}
7898	}
7899
7900
7901	/**
7902	* Exports the host state into the VMCS host-state area.
7903	* Sets up the VM-exit MSR-load area.
7904	*
7905	* The CPU state will be loaded from these fields on every successful VM-exit.
7906	*
7907	* @returns VBox status code.
7908	* @param pVCpu The cross context virtual CPU structure.
7909	*
7910	* @remarks No-long-jump zone!!!
7911	*/
7912	static int hmR0VmxExportHostState(PVMCPU pVCpu)
7913	{
7914	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7915
7916	int rc = VINF_SUCCESS;
7917	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7918	{
7919	rc = hmR0VmxExportHostControlRegs();
7920	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7921
7922	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7923	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7924
7925	rc = hmR0VmxExportHostMsrs(pVCpu);
7926	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7927
7928	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7929	}
7930	return rc;
7931	}
7932
7933
7934	/**
7935	* Saves the host state in the VMCS host-state.
7936	*
7937	* @returns VBox status code.
7938	* @param pVCpu The cross context virtual CPU structure.
7939	*
7940	* @remarks No-long-jump zone!!!
7941	*/
7942	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7943	{
7944	AssertPtr(pVCpu);
7945	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7946
7947	/*
7948	* Export the host state here while entering HM context.
7949	* When thread-context hooks are used, we might get preempted and have to re-save the host
7950	* state but most of the time we won't be, so do it here before we disable interrupts.
7951	*/
7952	return hmR0VmxExportHostState(pVCpu);
7953	}
7954
7955
7956	/**
7957	* Exports the guest state into the VMCS guest-state area.
7958	*
7959	* The will typically be done before VM-entry when the guest-CPU state and the
7960	* VMCS state may potentially be out of sync.
7961	*
7962	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7963	* VM-entry controls.
7964	* Sets up the appropriate VMX non-root function to execute guest code based on
7965	* the guest CPU mode.
7966	*
7967	* @returns VBox strict status code.
7968	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7969	* without unrestricted guest access and the VMMDev is not presently
7970	* mapped (e.g. EFI32).
7971	*
7972	* @param pVCpu The cross context virtual CPU structure.
7973	*
7974	* @remarks No-long-jump zone!!!
7975	*/
7976	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
7977	{
7978	AssertPtr(pVCpu);
7979	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7980
7981	LogFlowFunc(("pVCpu=%p\n", pVCpu));
7982
7983	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
7984
7985	/* Determine real-on-v86 mode. */
7986	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7987	if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
7988	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
7989	{
7990	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7991	}
7992
7993	/*
7994	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7995	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
7996	*/
7997	int rc = hmR0VmxSelectVMRunHandler(pVCpu);
7998	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7999
8000	/* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8001	rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8002	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8003
8004	/* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8005	rc = hmR0VmxExportGuestExitCtls(pVCpu);
8006	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8007
8008	rc = hmR0VmxExportGuestCR0(pVCpu);
8009	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8010
8011	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8012	if (rcStrict == VINF_SUCCESS)
8013	{ /* likely */ }
8014	else
8015	{
8016	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8017	return rcStrict;
8018	}
8019
8020	rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8021	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8022
8023	/* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8024	may alter controls if we determine we don't have to swap EFER after all. */
8025	rc = hmR0VmxExportGuestMsrs(pVCpu);
8026	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8027
8028	rc = hmR0VmxExportGuestApicTpr(pVCpu);
8029	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8030
8031	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8032	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8033
8034	/* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8035	not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8036	rc = hmR0VmxExportGuestRip(pVCpu);
8037	rc \|= hmR0VmxExportGuestRsp(pVCpu);
8038	rc \|= hmR0VmxExportGuestRflags(pVCpu);
8039	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8040
8041	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8042	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8043	\| HM_CHANGED_GUEST_CR2
8044	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8045	\| HM_CHANGED_GUEST_X87
8046	\| HM_CHANGED_GUEST_SSE_AVX
8047	\| HM_CHANGED_GUEST_OTHER_XSAVE
8048	\| HM_CHANGED_GUEST_XCRx
8049	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8050	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8051	\| HM_CHANGED_GUEST_TSC_AUX
8052	\| HM_CHANGED_GUEST_OTHER_MSRS
8053	\| HM_CHANGED_GUEST_HWVIRT
8054	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8055
8056	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8057	return rc;
8058	}
8059
8060
8061	/**
8062	* Exports the state shared between the host and guest into the VMCS.
8063	*
8064	* @param pVCpu The cross context virtual CPU structure.
8065	*
8066	* @remarks No-long-jump zone!!!
8067	*/
8068	static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8069	{
8070	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8071	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8072
8073	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8074	{
8075	int rc = hmR0VmxExportSharedDebugState(pVCpu);
8076	AssertRC(rc);
8077	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8078
8079	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8080	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8081	{
8082	rc = hmR0VmxExportGuestRflags(pVCpu);
8083	AssertRC(rc);
8084	}
8085	}
8086
8087	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8088	{
8089	hmR0VmxLazyLoadGuestMsrs(pVCpu);
8090	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8091	}
8092
8093	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8094	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8095	}
8096
8097
8098	/**
8099	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8100	*
8101	* @returns Strict VBox status code (i.e. informational status codes too).
8102	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8103	* without unrestricted guest access and the VMMDev is not presently
8104	* mapped (e.g. EFI32).
8105	*
8106	* @param pVCpu The cross context virtual CPU structure.
8107	*
8108	* @remarks No-long-jump zone!!!
8109	*/
8110	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8111	{
8112	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8113	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8114	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8115
8116	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8117	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8118	#endif
8119
8120	/*
8121	* For many exits it's only RIP that changes and hence try to export it first
8122	* without going through a lot of change flag checks.
8123	*/
8124	VBOXSTRICTRC rcStrict;
8125	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8126	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8127	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8128	{
8129	rcStrict = hmR0VmxExportGuestRip(pVCpu);
8130	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8131	{ /* likely */}
8132	else
8133	AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8134	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8135	}
8136	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8137	{
8138	rcStrict = hmR0VmxExportGuestState(pVCpu);
8139	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8140	{ /* likely */}
8141	else
8142	{
8143	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8144	VBOXSTRICTRC_VAL(rcStrict)));
8145	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8146	return rcStrict;
8147	}
8148	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8149	}
8150	else
8151	rcStrict = VINF_SUCCESS;
8152
8153	#ifdef VBOX_STRICT
8154	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8155	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8156	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8157	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8158	("fCtxChanged=%#RX64\n", fCtxChanged));
8159	#endif
8160	return rcStrict;
8161	}
8162
8163
8164	/**
8165	* Does the preparations before executing guest code in VT-x.
8166	*
8167	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8168	* recompiler/IEM. We must be cautious what we do here regarding committing
8169	* guest-state information into the VMCS assuming we assuredly execute the
8170	* guest in VT-x mode.
8171	*
8172	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8173	* the common-state (TRPM/forceflags), we must undo those changes so that the
8174	* recompiler/IEM can (and should) use them when it resumes guest execution.
8175	* Otherwise such operations must be done when we can no longer exit to ring-3.
8176	*
8177	* @returns Strict VBox status code (i.e. informational status codes too).
8178	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8179	* have been disabled.
8180	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8181	* double-fault into the guest.
8182	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8183	* dispatched directly.
8184	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8185	*
8186	* @param pVCpu The cross context virtual CPU structure.
8187	* @param pVmxTransient Pointer to the VMX transient structure.
8188	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8189	* us ignore some of the reasons for returning to
8190	* ring-3, and return VINF_EM_DBG_STEPPED if event
8191	* dispatching took place.
8192	*/
8193	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8194	{
8195	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8196
8197	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8198	PGMRZDynMapFlushAutoSet(pVCpu);
8199	#endif
8200
8201	/* Check force flag actions that might require us to go back to ring-3. */
8202	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8203	if (rcStrict == VINF_SUCCESS)
8204	{ /* FFs doesn't get set all the time. */ }
8205	else
8206	return rcStrict;
8207
8208	/*
8209	* Setup the virtualized-APIC accesses.
8210	*
8211	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8212	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8213	*
8214	* This is the reason we do it here and not in hmR0VmxExportGuestState().
8215	*/
8216	PVM pVM = pVCpu->CTX_SUFF(pVM);
8217	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8218	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8219	&& PDMHasApic(pVM))
8220	{
8221	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8222	Assert(u64MsrApicBase);
8223	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8224
8225	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8226
8227	/* Unalias any existing mapping. */
8228	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8229	AssertRCReturn(rc, rc);
8230
8231	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8232	Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8233	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8234	AssertRCReturn(rc, rc);
8235
8236	/* Update the per-VCPU cache of the APIC base MSR. */
8237	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8238	}
8239
8240	if (TRPMHasTrap(pVCpu))
8241	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8242	uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8243
8244	/*
8245	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8246	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8247	* also result in triple-faulting the VM.
8248	*/
8249	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8250	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8251	{ /* likely */ }
8252	else
8253	{
8254	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8255	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8256	return rcStrict;
8257	}
8258
8259	/*
8260	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
8261	* import CR3 themselves. We will need to update them here, as even as late as the above
8262	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8263	* the below force flags to be set.
8264	*/
8265	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8266	{
8267	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8268	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8269	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
8270	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8271	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8272	}
8273	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8274	{
8275	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8276	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8277	}
8278
8279	/*
8280	* No longjmps to ring-3 from this point on!!!
8281	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8282	* This also disables flushing of the R0-logger instance (if any).
8283	*/
8284	VMMRZCallRing3Disable(pVCpu);
8285
8286	/*
8287	* Export the guest state bits.
8288	*
8289	* We cannot perform longjmps while loading the guest state because we do not preserve the
8290	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8291	* CPU migration.
8292	*
8293	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8294	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8295	* Hence, loading of the guest state needs to be done -after- injection of events.
8296	*/
8297	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8298	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8299	{ /* likely */ }
8300	else
8301	{
8302	VMMRZCallRing3Enable(pVCpu);
8303	return rcStrict;
8304	}
8305
8306	/*
8307	* We disable interrupts so that we don't miss any interrupts that would flag preemption
8308	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8309	* preemption disabled for a while. Since this is purly to aid the
8310	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8311	* disable interrupt on NT.
8312	*
8313	* We need to check for force-flags that could've possible been altered since we last
8314	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8315	* see @bugref{6398}).
8316	*
8317	* We also check a couple of other force-flags as a last opportunity to get the EMT back
8318	* to ring-3 before executing guest code.
8319	*/
8320	pVmxTransient->fEFlags = ASMIntDisableFlags();
8321
8322	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8323	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8324	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8325	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8326	{
8327	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8328	{
8329	pVCpu->hm.s.Event.fPending = false;
8330
8331	/*
8332	* We've injected any pending events. This is really the point of no return (to ring-3).
8333	*
8334	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
8335	* returns from this function, so don't enable them here.
8336	*/
8337	return VINF_SUCCESS;
8338	}
8339
8340	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8341	rcStrict = VINF_EM_RAW_INTERRUPT;
8342	}
8343	else
8344	{
8345	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8346	rcStrict = VINF_EM_RAW_TO_R3;
8347	}
8348
8349	ASMSetFlags(pVmxTransient->fEFlags);
8350	VMMRZCallRing3Enable(pVCpu);
8351
8352	return rcStrict;
8353	}
8354
8355
8356	/**
8357	* Prepares to run guest code in VT-x and we've committed to doing so. This
8358	* means there is no backing out to ring-3 or anywhere else at this
8359	* point.
8360	*
8361	* @param pVCpu The cross context virtual CPU structure.
8362	* @param pVmxTransient Pointer to the VMX transient structure.
8363	*
8364	* @remarks Called with preemption disabled.
8365	* @remarks No-long-jump zone!!!
8366	*/
8367	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8368	{
8369	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8370	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8371	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8372
8373	/*
8374	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8375	*/
8376	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8377	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8378
8379	PVM pVM = pVCpu->CTX_SUFF(pVM);
8380	if (!CPUMIsGuestFPUStateActive(pVCpu))
8381	{
8382	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8383	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8384	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
8385	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8386	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8387	}
8388
8389	/*
8390	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8391	*/
8392	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8393	&& pVCpu->hm.s.vmx.cMsrs > 0)
8394	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8395
8396	/*
8397	* Re-save the host state bits as we may've been preempted (only happens when
8398	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8399	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8400	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
8401	* See @bugref{8432}.
8402	*/
8403	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8404	{
8405	int rc = hmR0VmxExportHostState(pVCpu);
8406	AssertRC(rc);
8407	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8408	}
8409	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8410
8411	/*
8412	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
8413	*/
8414	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8415	hmR0VmxExportSharedState(pVCpu);
8416	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8417
8418	/* Store status of the shared guest-host state at the time of VM-entry. */
8419	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8420	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8421	{
8422	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8423	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8424	}
8425	else
8426	#endif
8427	{
8428	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8429	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8430	}
8431
8432	/*
8433	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8434	*/
8435	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8436	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8437
8438	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8439	RTCPUID idCurrentCpu = pCpu->idCpu;
8440	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8441	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8442	{
8443	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8444	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8445	}
8446
8447	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8448	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8449	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8450	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8451
8452	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8453
8454	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8455	to start executing. */
8456
8457	/*
8458	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8459	*/
8460	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8461	{
8462	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8463	{
8464	bool fMsrUpdated;
8465	hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8466	int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8467	&fMsrUpdated);
8468	AssertRC(rc2);
8469	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8470	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8471	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8472	}
8473	else
8474	{
8475	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8476	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8477	}
8478	}
8479
8480	if (pVM->cpum.ro.GuestFeatures.fIbrs)
8481	{
8482	bool fMsrUpdated;
8483	hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8484	int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8485	&fMsrUpdated);
8486	AssertRC(rc2);
8487	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8488	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8489	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8490	}
8491
8492	#ifdef VBOX_STRICT
8493	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8494	hmR0VmxCheckHostEferMsr(pVCpu);
8495	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8496	#endif
8497	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8498	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8499	{
8500	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8501	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8502	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8503	}
8504	#endif
8505	}
8506
8507
8508	/**
8509	* Performs some essential restoration of state after running guest code in
8510	* VT-x.
8511	*
8512	* @param pVCpu The cross context virtual CPU structure.
8513	* @param pVmxTransient Pointer to the VMX transient structure.
8514	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8515	*
8516	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
8517	*
8518	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8519	* unconditionally when it is safe to do so.
8520	*/
8521	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8522	{
8523	uint64_t const uHostTsc = ASMReadTSC();
8524	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8525
8526	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8527	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8528	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8529	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8530	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8531	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8532
8533	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8534	TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8535
8536	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8537	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8538	Assert(!ASMIntAreEnabled());
8539	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8540
8541	#if HC_ARCH_BITS == 64
8542	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8543	#endif
8544	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8545	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8546	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8547	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8548	#else
8549	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8550	#endif
8551	#ifdef VBOX_STRICT
8552	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8553	#endif
8554	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8555
8556	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8557	uint32_t uExitReason;
8558	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8559	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8560	AssertRC(rc);
8561	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8562	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8563
8564	if (rcVMRun == VINF_SUCCESS)
8565	{
8566	/*
8567	* Update the VM-exit history array here even if the VM-entry failed due to:
8568	* - Invalid guest state.
8569	* - MSR loading.
8570	* - Machine-check event.
8571	*
8572	* In any of the above cases we will still have a "valid" VM-exit reason
8573	* despite @a fVMEntryFailed being false.
8574	*
8575	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8576	*
8577	* Note! We don't have CS or RIP at this point. Will probably address that later
8578	* by amending the history entry added here.
8579	*/
8580	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8581	UINT64_MAX, uHostTsc);
8582
8583	if (!pVmxTransient->fVMEntryFailed)
8584	{
8585	VMMRZCallRing3Enable(pVCpu);
8586
8587	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8588	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8589
8590	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8591	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8592	AssertRC(rc);
8593	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8594	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8595	AssertRC(rc);
8596	#else
8597	/*
8598	* Import the guest-interruptibility state always as we need it while evaluating
8599	* injecting events on re-entry.
8600	*
8601	* We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8602	* checking for real-mode while exporting the state because all bits that cause
8603	* mode changes wrt CR0 are intercepted.
8604	*/
8605	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8606	AssertRC(rc);
8607	#endif
8608
8609	/*
8610	* Sync the TPR shadow with our APIC state.
8611	*/
8612	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8613	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8614	{
8615	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8616	AssertRC(rc);
8617	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8618	}
8619
8620	return;
8621	}
8622	}
8623	else
8624	{
8625	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8626	}
8627
8628	VMMRZCallRing3Enable(pVCpu);
8629	}
8630
8631
8632	/**
8633	* Runs the guest code using VT-x the normal way.
8634	*
8635	* @returns VBox status code.
8636	* @param pVCpu The cross context virtual CPU structure.
8637	*
8638	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8639	*/
8640	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8641	{
8642	VMXTRANSIENT VmxTransient;
8643	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8644	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8645	uint32_t cLoops = 0;
8646
8647	for (;; cLoops++)
8648	{
8649	Assert(!HMR0SuspendPending());
8650	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8651
8652	/* Preparatory work for running guest code, this may force us to return
8653	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8654	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8655	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8656	if (rcStrict != VINF_SUCCESS)
8657	break;
8658
8659	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8660	int rcRun = hmR0VmxRunGuest(pVCpu);
8661
8662	/* Restore any residual host-state and save any bits shared between host
8663	and guest into the guest-CPU state. Re-enables interrupts! */
8664	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8665
8666	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8667	if (RT_SUCCESS(rcRun))
8668	{ /* very likely */ }
8669	else
8670	{
8671	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8672	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8673	return rcRun;
8674	}
8675
8676	/* Profile the VM-exit. */
8677	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8679	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8680	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8681	HMVMX_START_EXIT_DISPATCH_PROF();
8682
8683	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8684
8685	/* Handle the VM-exit. */
8686	#ifdef HMVMX_USE_FUNCTION_TABLE
8687	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8688	#else
8689	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8690	#endif
8691	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8692	if (rcStrict == VINF_SUCCESS)
8693	{
8694	if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8695	continue; /* likely */
8696	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8697	rcStrict = VINF_EM_RAW_INTERRUPT;
8698	}
8699	break;
8700	}
8701
8702	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8703	return rcStrict;
8704	}
8705
8706
8707
8708	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8709	* probes.
8710	*
8711	* The following few functions and associated structure contains the bloat
8712	* necessary for providing detailed debug events and dtrace probes as well as
8713	* reliable host side single stepping. This works on the principle of
8714	* "subclassing" the normal execution loop and workers. We replace the loop
8715	* method completely and override selected helpers to add necessary adjustments
8716	* to their core operation.
8717	*
8718	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8719	* any performance for debug and analysis features.
8720	*
8721	* @{
8722	*/
8723
8724	/**
8725	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
8726	* the debug run loop.
8727	*/
8728	typedef struct VMXRUNDBGSTATE
8729	{
8730	/** The RIP we started executing at. This is for detecting that we stepped. */
8731	uint64_t uRipStart;
8732	/** The CS we started executing with. */
8733	uint16_t uCsStart;
8734
8735	/** Whether we've actually modified the 1st execution control field. */
8736	bool fModifiedProcCtls : 1;
8737	/** Whether we've actually modified the 2nd execution control field. */
8738	bool fModifiedProcCtls2 : 1;
8739	/** Whether we've actually modified the exception bitmap. */
8740	bool fModifiedXcptBitmap : 1;
8741
8742	/** We desire the modified the CR0 mask to be cleared. */
8743	bool fClearCr0Mask : 1;
8744	/** We desire the modified the CR4 mask to be cleared. */
8745	bool fClearCr4Mask : 1;
8746	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8747	uint32_t fCpe1Extra;
8748	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8749	uint32_t fCpe1Unwanted;
8750	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8751	uint32_t fCpe2Extra;
8752	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8753	uint32_t bmXcptExtra;
8754	/** The sequence number of the Dtrace provider settings the state was
8755	* configured against. */
8756	uint32_t uDtraceSettingsSeqNo;
8757	/** VM-exits to check (one bit per VM-exit). */
8758	uint32_t bmExitsToCheck[3];
8759
8760	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8761	uint32_t fProcCtlsInitial;
8762	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8763	uint32_t fProcCtls2Initial;
8764	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8765	uint32_t bmXcptInitial;
8766	} VMXRUNDBGSTATE;
8767	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8768	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8769
8770
8771	/**
8772	* Initializes the VMXRUNDBGSTATE structure.
8773	*
8774	* @param pVCpu The cross context virtual CPU structure of the
8775	* calling EMT.
8776	* @param pDbgState The structure to initialize.
8777	*/
8778	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8779	{
8780	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8781	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8782
8783	pDbgState->fModifiedProcCtls = false;
8784	pDbgState->fModifiedProcCtls2 = false;
8785	pDbgState->fModifiedXcptBitmap = false;
8786	pDbgState->fClearCr0Mask = false;
8787	pDbgState->fClearCr4Mask = false;
8788	pDbgState->fCpe1Extra = 0;
8789	pDbgState->fCpe1Unwanted = 0;
8790	pDbgState->fCpe2Extra = 0;
8791	pDbgState->bmXcptExtra = 0;
8792	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8793	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8794	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8795	}
8796
8797
8798	/**
8799	* Updates the VMSC fields with changes requested by @a pDbgState.
8800	*
8801	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8802	* immediately before executing guest code, i.e. when interrupts are disabled.
8803	* We don't check status codes here as we cannot easily assert or return in the
8804	* latter case.
8805	*
8806	* @param pVCpu The cross context virtual CPU structure.
8807	* @param pDbgState The debug state.
8808	*/
8809	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8810	{
8811	/*
8812	* Ensure desired flags in VMCS control fields are set.
8813	* (Ignoring write failure here, as we're committed and it's just debug extras.)
8814	*
8815	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8816	* there should be no stale data in pCtx at this point.
8817	*/
8818	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8819	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8820	{
8821	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
8822	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8823	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8824	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8825	pDbgState->fModifiedProcCtls = true;
8826	}
8827
8828	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8829	{
8830	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
8831	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8832	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8833	pDbgState->fModifiedProcCtls2 = true;
8834	}
8835
8836	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8837	{
8838	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
8839	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8840	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8841	pDbgState->fModifiedXcptBitmap = true;
8842	}
8843
8844	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8845	{
8846	pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8847	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8848	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8849	}
8850
8851	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8852	{
8853	pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8854	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8855	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8856	}
8857	}
8858
8859
8860	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8861	{
8862	/*
8863	* Restore VM-exit control settings as we may not reenter this function the
8864	* next time around.
8865	*/
8866	/* We reload the initial value, trigger what we can of recalculations the
8867	next time around. From the looks of things, that's all that's required atm. */
8868	if (pDbgState->fModifiedProcCtls)
8869	{
8870	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8871	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8872	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8873	AssertRCReturn(rc2, rc2);
8874	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8875	}
8876
8877	/* We're currently the only ones messing with this one, so just restore the
8878	cached value and reload the field. */
8879	if ( pDbgState->fModifiedProcCtls2
8880	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8881	{
8882	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8883	AssertRCReturn(rc2, rc2);
8884	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8885	}
8886
8887	/* If we've modified the exception bitmap, we restore it and trigger
8888	reloading and partial recalculation the next time around. */
8889	if (pDbgState->fModifiedXcptBitmap)
8890	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8891
8892	return rcStrict;
8893	}
8894
8895
8896	/**
8897	* Configures VM-exit controls for current DBGF and DTrace settings.
8898	*
8899	* This updates @a pDbgState and the VMCS execution control fields to reflect
8900	* the necessary VM-exits demanded by DBGF and DTrace.
8901	*
8902	* @param pVCpu The cross context virtual CPU structure.
8903	* @param pDbgState The debug state.
8904	* @param pVmxTransient Pointer to the VMX transient structure. May update
8905	* fUpdateTscOffsettingAndPreemptTimer.
8906	*/
8907	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8908	{
8909	/*
8910	* Take down the dtrace serial number so we can spot changes.
8911	*/
8912	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8913	ASMCompilerBarrier();
8914
8915	/*
8916	* We'll rebuild most of the middle block of data members (holding the
8917	* current settings) as we go along here, so start by clearing it all.
8918	*/
8919	pDbgState->bmXcptExtra = 0;
8920	pDbgState->fCpe1Extra = 0;
8921	pDbgState->fCpe1Unwanted = 0;
8922	pDbgState->fCpe2Extra = 0;
8923	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8924	pDbgState->bmExitsToCheck[i] = 0;
8925
8926	/*
8927	* Software interrupts (INT XXh) - no idea how to trigger these...
8928	*/
8929	PVM pVM = pVCpu->CTX_SUFF(pVM);
8930	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8931	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
8932	{
8933	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8934	}
8935
8936	/*
8937	* INT3 breakpoints - triggered by #BP exceptions.
8938	*/
8939	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8940	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
8941
8942	/*
8943	* Exception bitmap and XCPT events+probes.
8944	*/
8945	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8946	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8947	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
8948
8949	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
8950	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
8951	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
8952	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
8953	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
8954	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
8955	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
8956	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
8957	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
8958	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
8959	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
8960	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
8961	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
8962	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
8963	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
8964	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
8965	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
8966	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
8967
8968	if (pDbgState->bmXcptExtra)
8969	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8970
8971	/*
8972	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8973	*
8974	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8975	* So, when adding/changing/removing please don't forget to update it.
8976	*
8977	* Some of the macros are picking up local variables to save horizontal space,
8978	* (being able to see it in a table is the lesser evil here).
8979	*/
8980	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8981	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8982	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8983	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8984	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8985	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8986	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8987	} else do { } while (0)
8988	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8989	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8990	{ \
8991	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
8992	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8993	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8994	} else do { } while (0)
8995	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
8996	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8997	{ \
8998	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
8999	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9000	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9001	} else do { } while (0)
9002	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9003	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9004	{ \
9005	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9006	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9007	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9008	} else do { } while (0)
9009
9010	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9011	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9012	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9013	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9014	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9015
9016	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9017	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9018	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9019	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9020	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9021	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9022	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9023	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9024	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9025	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9026	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9027	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9028	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9029	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9030	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9031	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9032	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9033	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9034	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9035	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9036	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9037	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9038	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9039	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9040	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9041	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9042	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9043	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9044	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9045	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9046	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9047	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9048	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9049	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9050	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9051	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9052
9053	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9054	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9055	{
9056	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_APIC_TPR);
9057	AssertRC(rc);
9058
9059	#if 0 /** @todo fix me */
9060	pDbgState->fClearCr0Mask = true;
9061	pDbgState->fClearCr4Mask = true;
9062	#endif
9063	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9064	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9065	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9066	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9067	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9068	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9069	require clearing here and in the loop if we start using it. */
9070	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9071	}
9072	else
9073	{
9074	if (pDbgState->fClearCr0Mask)
9075	{
9076	pDbgState->fClearCr0Mask = false;
9077	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9078	}
9079	if (pDbgState->fClearCr4Mask)
9080	{
9081	pDbgState->fClearCr4Mask = false;
9082	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9083	}
9084	}
9085	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9086	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9087
9088	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9089	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9090	{
9091	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9092	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9093	}
9094	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9095	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9096
9097	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9098	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9099	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9100	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9101	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9102	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9103	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9104	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9105	#if 0 /** @todo too slow, fix handler. */
9106	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9107	#endif
9108	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9109
9110	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9111	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9112	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9113	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9114	{
9115	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9116	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9117	}
9118	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9119	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9120	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9121	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9122
9123	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9124	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9125	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9126	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9127	{
9128	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9129	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9130	}
9131	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9132	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9133	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9134	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9135
9136	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9137	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9138	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9139	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9140	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9141	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9142	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9143	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9144	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9145	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9146	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9147	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9148	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9149	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9150	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9151	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9152	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9153	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9154	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9155	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9156	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9157	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9158
9159	#undef IS_EITHER_ENABLED
9160	#undef SET_ONLY_XBM_IF_EITHER_EN
9161	#undef SET_CPE1_XBM_IF_EITHER_EN
9162	#undef SET_CPEU_XBM_IF_EITHER_EN
9163	#undef SET_CPE2_XBM_IF_EITHER_EN
9164
9165	/*
9166	* Sanitize the control stuff.
9167	*/
9168	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9169	if (pDbgState->fCpe2Extra)
9170	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9171	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9172	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9173	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9174	{
9175	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9176	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9177	}
9178
9179	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9180	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9181	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9182	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9183	}
9184
9185
9186	/**
9187	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9188	* appropriate.
9189	*
9190	* The caller has checked the VM-exit against the
9191	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9192	* already, so we don't have to do that either.
9193	*
9194	* @returns Strict VBox status code (i.e. informational status codes too).
9195	* @param pVCpu The cross context virtual CPU structure.
9196	* @param pVmxTransient Pointer to the VMX-transient structure.
9197	* @param uExitReason The VM-exit reason.
9198	*
9199	* @remarks The name of this function is displayed by dtrace, so keep it short
9200	* and to the point. No longer than 33 chars long, please.
9201	*/
9202	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9203	{
9204	/*
9205	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9206	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9207	*
9208	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9209	* does. Must add/change/remove both places. Same ordering, please.
9210	*
9211	* Added/removed events must also be reflected in the next section
9212	* where we dispatch dtrace events.
9213	*/
9214	bool fDtrace1 = false;
9215	bool fDtrace2 = false;
9216	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9217	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9218	uint32_t uEventArg = 0;
9219	#define SET_EXIT(a_EventSubName) \
9220	do { \
9221	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9222	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9223	} while (0)
9224	#define SET_BOTH(a_EventSubName) \
9225	do { \
9226	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9227	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9228	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9229	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9230	} while (0)
9231	switch (uExitReason)
9232	{
9233	case VMX_EXIT_MTF:
9234	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9235
9236	case VMX_EXIT_XCPT_OR_NMI:
9237	{
9238	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9239	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9240	{
9241	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9242	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9243	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9244	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9245	{
9246	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9247	{
9248	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9249	uEventArg = pVmxTransient->uExitIntErrorCode;
9250	}
9251	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9252	switch (enmEvent1)
9253	{
9254	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9255	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9256	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9257	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9258	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9259	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9260	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9261	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9262	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9263	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9264	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9265	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9266	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9267	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9268	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9269	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9270	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9271	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9272	default: break;
9273	}
9274	}
9275	else
9276	AssertFailed();
9277	break;
9278
9279	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9280	uEventArg = idxVector;
9281	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9282	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9283	break;
9284	}
9285	break;
9286	}
9287
9288	case VMX_EXIT_TRIPLE_FAULT:
9289	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9290	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9291	break;
9292	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9293	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9294	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9295	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9296	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9297
9298	/* Instruction specific VM-exits: */
9299	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9300	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9301	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9302	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9303	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9304	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9305	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9306	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9307	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9308	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9309	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9310	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9311	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9312	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9313	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9314	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9315	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9316	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9317	case VMX_EXIT_MOV_CRX:
9318	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9319	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9320	SET_BOTH(CRX_READ);
9321	else
9322	SET_BOTH(CRX_WRITE);
9323	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9324	break;
9325	case VMX_EXIT_MOV_DRX:
9326	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9327	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9328	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9329	SET_BOTH(DRX_READ);
9330	else
9331	SET_BOTH(DRX_WRITE);
9332	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9333	break;
9334	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9335	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9336	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9337	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9338	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9339	case VMX_EXIT_XDTR_ACCESS:
9340	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9341	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9342	{
9343	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9344	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9345	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9346	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9347	}
9348	break;
9349
9350	case VMX_EXIT_TR_ACCESS:
9351	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9352	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9353	{
9354	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9355	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9356	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9357	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9358	}
9359	break;
9360
9361	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9362	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9363	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9364	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9365	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9366	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9367	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9368	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9369	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9370	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9371	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9372
9373	/* Events that aren't relevant at this point. */
9374	case VMX_EXIT_EXT_INT:
9375	case VMX_EXIT_INT_WINDOW:
9376	case VMX_EXIT_NMI_WINDOW:
9377	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9378	case VMX_EXIT_PREEMPT_TIMER:
9379	case VMX_EXIT_IO_INSTR:
9380	break;
9381
9382	/* Errors and unexpected events. */
9383	case VMX_EXIT_INIT_SIGNAL:
9384	case VMX_EXIT_SIPI:
9385	case VMX_EXIT_IO_SMI:
9386	case VMX_EXIT_SMI:
9387	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9388	case VMX_EXIT_ERR_MSR_LOAD:
9389	case VMX_EXIT_ERR_MACHINE_CHECK:
9390	break;
9391
9392	default:
9393	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9394	break;
9395	}
9396	#undef SET_BOTH
9397	#undef SET_EXIT
9398
9399	/*
9400	* Dtrace tracepoints go first. We do them here at once so we don't
9401	* have to copy the guest state saving and stuff a few dozen times.
9402	* Down side is that we've got to repeat the switch, though this time
9403	* we use enmEvent since the probes are a subset of what DBGF does.
9404	*/
9405	if (fDtrace1 \|\| fDtrace2)
9406	{
9407	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9408	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9409	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9410	switch (enmEvent1)
9411	{
9412	/** @todo consider which extra parameters would be helpful for each probe. */
9413	case DBGFEVENT_END: break;
9414	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9415	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9416	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9417	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9418	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9419	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9420	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9421	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9422	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9423	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9424	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9425	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9426	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9427	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9428	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9429	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9430	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9431	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9432	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9433	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9434	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9435	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9436	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9437	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9438	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9439	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9440	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9441	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9442	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9443	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9444	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9445	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9446	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9447	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9448	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9449	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9450	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9451	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9452	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9453	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9454	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9455	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9456	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9457	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9458	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9459	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9460	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9461	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9462	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9463	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9464	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9465	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9466	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9467	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9468	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9469	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9470	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9471	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9472	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9473	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9474	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9475	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9476	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9477	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9478	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9479	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9480	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9481	}
9482	switch (enmEvent2)
9483	{
9484	/** @todo consider which extra parameters would be helpful for each probe. */
9485	case DBGFEVENT_END: break;
9486	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9487	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9488	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9489	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9490	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9491	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9492	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9493	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9494	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9495	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9496	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9497	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9498	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9499	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9500	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9501	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9502	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9503	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9504	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9505	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9506	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9507	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9508	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9509	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9510	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9511	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9512	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9513	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9514	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9515	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9516	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9517	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9518	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9519	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9520	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9521	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9522	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9523	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9524	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9525	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9526	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9527	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9528	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9529	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9530	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9531	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9532	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9533	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9534	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9535	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9536	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9537	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9538	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9539	}
9540	}
9541
9542	/*
9543	* Fire of the DBGF event, if enabled (our check here is just a quick one,
9544	* the DBGF call will do a full check).
9545	*
9546	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9547	* Note! If we have to events, we prioritize the first, i.e. the instruction
9548	* one, in order to avoid event nesting.
9549	*/
9550	PVM pVM = pVCpu->CTX_SUFF(pVM);
9551	if ( enmEvent1 != DBGFEVENT_END
9552	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9553	{
9554	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9555	if (rcStrict != VINF_SUCCESS)
9556	return rcStrict;
9557	}
9558	else if ( enmEvent2 != DBGFEVENT_END
9559	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9560	{
9561	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9562	if (rcStrict != VINF_SUCCESS)
9563	return rcStrict;
9564	}
9565
9566	return VINF_SUCCESS;
9567	}
9568
9569
9570	/**
9571	* Single-stepping VM-exit filtering.
9572	*
9573	* This is preprocessing the VM-exits and deciding whether we've gotten far
9574	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9575	* handling is performed.
9576	*
9577	* @returns Strict VBox status code (i.e. informational status codes too).
9578	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9579	* @param pVmxTransient Pointer to the VMX-transient structure.
9580	* @param pDbgState The debug state.
9581	*/
9582	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9583	{
9584	/*
9585	* Expensive (saves context) generic dtrace VM-exit probe.
9586	*/
9587	uint32_t const uExitReason = pVmxTransient->uExitReason;
9588	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9589	{ /* more likely */ }
9590	else
9591	{
9592	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9593	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9594	AssertRC(rc);
9595	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9596	}
9597
9598	/*
9599	* Check for host NMI, just to get that out of the way.
9600	*/
9601	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9602	{ /* normally likely */ }
9603	else
9604	{
9605	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9606	AssertRCReturn(rc2, rc2);
9607	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9608	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9609	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9610	}
9611
9612	/*
9613	* Check for single stepping event if we're stepping.
9614	*/
9615	if (pVCpu->hm.s.fSingleInstruction)
9616	{
9617	switch (uExitReason)
9618	{
9619	case VMX_EXIT_MTF:
9620	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9621
9622	/* Various events: */
9623	case VMX_EXIT_XCPT_OR_NMI:
9624	case VMX_EXIT_EXT_INT:
9625	case VMX_EXIT_TRIPLE_FAULT:
9626	case VMX_EXIT_INT_WINDOW:
9627	case VMX_EXIT_NMI_WINDOW:
9628	case VMX_EXIT_TASK_SWITCH:
9629	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9630	case VMX_EXIT_APIC_ACCESS:
9631	case VMX_EXIT_EPT_VIOLATION:
9632	case VMX_EXIT_EPT_MISCONFIG:
9633	case VMX_EXIT_PREEMPT_TIMER:
9634
9635	/* Instruction specific VM-exits: */
9636	case VMX_EXIT_CPUID:
9637	case VMX_EXIT_GETSEC:
9638	case VMX_EXIT_HLT:
9639	case VMX_EXIT_INVD:
9640	case VMX_EXIT_INVLPG:
9641	case VMX_EXIT_RDPMC:
9642	case VMX_EXIT_RDTSC:
9643	case VMX_EXIT_RSM:
9644	case VMX_EXIT_VMCALL:
9645	case VMX_EXIT_VMCLEAR:
9646	case VMX_EXIT_VMLAUNCH:
9647	case VMX_EXIT_VMPTRLD:
9648	case VMX_EXIT_VMPTRST:
9649	case VMX_EXIT_VMREAD:
9650	case VMX_EXIT_VMRESUME:
9651	case VMX_EXIT_VMWRITE:
9652	case VMX_EXIT_VMXOFF:
9653	case VMX_EXIT_VMXON:
9654	case VMX_EXIT_MOV_CRX:
9655	case VMX_EXIT_MOV_DRX:
9656	case VMX_EXIT_IO_INSTR:
9657	case VMX_EXIT_RDMSR:
9658	case VMX_EXIT_WRMSR:
9659	case VMX_EXIT_MWAIT:
9660	case VMX_EXIT_MONITOR:
9661	case VMX_EXIT_PAUSE:
9662	case VMX_EXIT_XDTR_ACCESS:
9663	case VMX_EXIT_TR_ACCESS:
9664	case VMX_EXIT_INVEPT:
9665	case VMX_EXIT_RDTSCP:
9666	case VMX_EXIT_INVVPID:
9667	case VMX_EXIT_WBINVD:
9668	case VMX_EXIT_XSETBV:
9669	case VMX_EXIT_RDRAND:
9670	case VMX_EXIT_INVPCID:
9671	case VMX_EXIT_VMFUNC:
9672	case VMX_EXIT_RDSEED:
9673	case VMX_EXIT_XSAVES:
9674	case VMX_EXIT_XRSTORS:
9675	{
9676	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9677	AssertRCReturn(rc, rc);
9678	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9679	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9680	return VINF_EM_DBG_STEPPED;
9681	break;
9682	}
9683
9684	/* Errors and unexpected events: */
9685	case VMX_EXIT_INIT_SIGNAL:
9686	case VMX_EXIT_SIPI:
9687	case VMX_EXIT_IO_SMI:
9688	case VMX_EXIT_SMI:
9689	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9690	case VMX_EXIT_ERR_MSR_LOAD:
9691	case VMX_EXIT_ERR_MACHINE_CHECK:
9692	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9693	break;
9694
9695	default:
9696	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9697	break;
9698	}
9699	}
9700
9701	/*
9702	* Check for debugger event breakpoints and dtrace probes.
9703	*/
9704	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9705	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9706	{
9707	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9708	if (rcStrict != VINF_SUCCESS)
9709	return rcStrict;
9710	}
9711
9712	/*
9713	* Normal processing.
9714	*/
9715	#ifdef HMVMX_USE_FUNCTION_TABLE
9716	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9717	#else
9718	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9719	#endif
9720	}
9721
9722
9723	/**
9724	* Single steps guest code using VT-x.
9725	*
9726	* @returns Strict VBox status code (i.e. informational status codes too).
9727	* @param pVCpu The cross context virtual CPU structure.
9728	*
9729	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9730	*/
9731	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9732	{
9733	VMXTRANSIENT VmxTransient;
9734	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9735
9736	/* Set HMCPU indicators. */
9737	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9738	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
9739	pVCpu->hm.s.fDebugWantRdTscExit = false;
9740	pVCpu->hm.s.fUsingDebugLoop = true;
9741
9742	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9743	VMXRUNDBGSTATE DbgState;
9744	hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
9745	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9746
9747	/*
9748	* The loop.
9749	*/
9750	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9751	for (uint32_t cLoops = 0; ; cLoops++)
9752	{
9753	Assert(!HMR0SuspendPending());
9754	HMVMX_ASSERT_CPU_SAFE(pVCpu);
9755	bool fStepping = pVCpu->hm.s.fSingleInstruction;
9756
9757	/*
9758	* Preparatory work for running guest code, this may force us to return
9759	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9760	*/
9761	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9762	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9763	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
9764	if (rcStrict != VINF_SUCCESS)
9765	break;
9766
9767	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
9768	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9769
9770	/*
9771	* Now we can run the guest code.
9772	*/
9773	int rcRun = hmR0VmxRunGuest(pVCpu);
9774
9775	/*
9776	* Restore any residual host-state and save any bits shared between host
9777	* and guest into the guest-CPU state. Re-enables interrupts!
9778	*/
9779	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9780
9781	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9782	if (RT_SUCCESS(rcRun))
9783	{ /* very likely */ }
9784	else
9785	{
9786	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9787	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9788	return rcRun;
9789	}
9790
9791	/* Profile the VM-exit. */
9792	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9793	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9794	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9795	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9796	HMVMX_START_EXIT_DISPATCH_PROF();
9797
9798	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9799
9800	/*
9801	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9802	*/
9803	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9804	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9805	if (rcStrict != VINF_SUCCESS)
9806	break;
9807	if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9808	{
9809	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9810	rcStrict = VINF_EM_RAW_INTERRUPT;
9811	break;
9812	}
9813
9814	/*
9815	* Stepping: Did the RIP change, if so, consider it a single step.
9816	* Otherwise, make sure one of the TFs gets set.
9817	*/
9818	if (fStepping)
9819	{
9820	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
9821	AssertRC(rc);
9822	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9823	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9824	{
9825	rcStrict = VINF_EM_DBG_STEPPED;
9826	break;
9827	}
9828	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9829	}
9830
9831	/*
9832	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
9833	*/
9834	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9835	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9836	}
9837
9838	/*
9839	* Clear the X86_EFL_TF if necessary.
9840	*/
9841	if (pVCpu->hm.s.fClearTrapFlag)
9842	{
9843	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9844	AssertRC(rc);
9845	pVCpu->hm.s.fClearTrapFlag = false;
9846	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9847	}
9848	/** @todo there seems to be issues with the resume flag when the monitor trap
9849	* flag is pending without being used. Seen early in bios init when
9850	* accessing APIC page in protected mode. */
9851
9852	/*
9853	* Restore VM-exit control settings as we may not reenter this function the
9854	* next time around.
9855	*/
9856	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9857
9858	/* Restore HMCPU indicators. */
9859	pVCpu->hm.s.fUsingDebugLoop = false;
9860	pVCpu->hm.s.fDebugWantRdTscExit = false;
9861	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9862
9863	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9864	return rcStrict;
9865	}
9866
9867
9868	/** @} */
9869
9870
9871	/**
9872	* Checks if any expensive dtrace probes are enabled and we should go to the
9873	* debug loop.
9874	*
9875	* @returns true if we should use debug loop, false if not.
9876	*/
9877	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9878	{
9879	/* It's probably faster to OR the raw 32-bit counter variables together.
9880	Since the variables are in an array and the probes are next to one
9881	another (more or less), we have good locality. So, better read
9882	eight-nine cache lines ever time and only have one conditional, than
9883	128+ conditionals, right? */
9884	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9885	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
9886	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
9887	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
9888	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
9889	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
9890	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
9891	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
9892	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
9893	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
9894	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
9895	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
9896	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
9897	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
9898	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
9899	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
9900	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
9901	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
9902	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
9903	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9904	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9905	) != 0
9906	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9907	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9908	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9909	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9910	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
9911	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9912	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9913	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9914	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9915	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9916	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9917	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9918	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9919	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9920	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9921	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9922	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9923	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9924	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9925	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9926	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9927	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9928	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9929	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9930	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
9931	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
9932	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9933	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
9934	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9935	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9936	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9937	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9938	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9939	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9940	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9941	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9942	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9943	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9944	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9945	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9946	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9947	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9948	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9949	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9950	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9951	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9952	) != 0
9953	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9954	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
9955	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9956	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9957	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9958	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
9959	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9960	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9961	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
9962	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
9963	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
9964	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
9965	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
9966	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
9967	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
9968	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
9969	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
9970	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
9971	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
9972	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
9973	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
9974	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
9975	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
9976	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
9977	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
9978	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
9979	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
9980	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
9981	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
9982	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
9983	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
9984	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
9985	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
9986	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
9987	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
9988	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
9989	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
9990	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
9991	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
9992	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
9993	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
9994	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
9995	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
9996	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
9997	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
9998	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
9999	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10000	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10001	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10002	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10003	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10004	) != 0;
10005	}
10006
10007
10008	/**
10009	* Runs the guest code using VT-x.
10010	*
10011	* @returns Strict VBox status code (i.e. informational status codes too).
10012	* @param pVCpu The cross context virtual CPU structure.
10013	*/
10014	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10015	{
10016	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10017	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10018	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10019	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10020
10021	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10022
10023	VBOXSTRICTRC rcStrict;
10024	if ( !pVCpu->hm.s.fUseDebugLoop
10025	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10026	&& !DBGFIsStepping(pVCpu)
10027	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10028	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10029	else
10030	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10031
10032	if (rcStrict == VERR_EM_INTERPRETER)
10033	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10034	else if (rcStrict == VINF_EM_RESET)
10035	rcStrict = VINF_EM_TRIPLE_FAULT;
10036
10037	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10038	if (RT_FAILURE(rc2))
10039	{
10040	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10041	rcStrict = rc2;
10042	}
10043	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10044	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10045	return rcStrict;
10046	}
10047
10048
10049	#ifndef HMVMX_USE_FUNCTION_TABLE
10050	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10051	{
10052	#ifdef DEBUG_ramshankar
10053	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10054	do { \
10055	if (a_fSave != 0) \
10056	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10057	VBOXSTRICTRC rcStrict = a_CallExpr; \
10058	if (a_fSave != 0) \
10059	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10060	return rcStrict; \
10061	} while (0)
10062	#else
10063	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10064	#endif
10065	switch (rcReason)
10066	{
10067	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10068	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10069	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10070	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10071	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10072	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10073	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10074	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10075	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10076	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10077	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10078	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10079	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10080	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10081	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10082	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10083	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10084	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10085	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10086	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10087	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10088	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10089	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10090	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10091	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10092	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10093	case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10094	case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10095	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10096	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10097	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10098	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10099	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10100	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10101
10102	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10103	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10104	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10105	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10106	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10107	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10108	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10109	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10110	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10111
10112	case VMX_EXIT_VMCLEAR:
10113	case VMX_EXIT_VMLAUNCH:
10114	case VMX_EXIT_VMPTRLD:
10115	case VMX_EXIT_VMPTRST:
10116	case VMX_EXIT_VMREAD:
10117	case VMX_EXIT_VMRESUME:
10118	case VMX_EXIT_VMWRITE:
10119	case VMX_EXIT_VMXOFF:
10120	case VMX_EXIT_VMXON:
10121	case VMX_EXIT_INVEPT:
10122	case VMX_EXIT_INVVPID:
10123	case VMX_EXIT_VMFUNC:
10124	case VMX_EXIT_XSAVES:
10125	case VMX_EXIT_XRSTORS:
10126	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10127
10128	case VMX_EXIT_ENCLS:
10129	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10130	case VMX_EXIT_PML_FULL:
10131	default:
10132	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10133	}
10134	#undef VMEXIT_CALL_RET
10135	}
10136	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10137
10138
10139	#ifdef VBOX_STRICT
10140	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10141	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10142	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10143
10144	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10145	do { \
10146	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10147	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10148	} while (0)
10149
10150	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10151	do { \
10152	AssertPtr((a_pVCpu)); \
10153	AssertPtr((a_pVmxTransient)); \
10154	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10155	Assert(ASMIntAreEnabled()); \
10156	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10157	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10158	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10159	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10160	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10161	HMVMX_ASSERT_PREEMPT_CPUID(); \
10162	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10163	} while (0)
10164
10165	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10166	do { \
10167	Log4Func(("\n")); \
10168	} while (0)
10169	#else
10170	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10171	do { \
10172	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10173	RT_NOREF2((a_pVCpu), (a_pVmxTransient)); \
10174	} while (0)
10175	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10176	#endif
10177
10178
10179	/**
10180	* Advances the guest RIP by the specified number of bytes.
10181	*
10182	* @param pVCpu The cross context virtual CPU structure.
10183	* @param cbInstr Number of bytes to advance the RIP by.
10184	*
10185	* @remarks No-long-jump zone!!!
10186	*/
10187	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10188	{
10189	/* Advance the RIP. */
10190	pVCpu->cpum.GstCtx.rip += cbInstr;
10191	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10192
10193	/* Update interrupt inhibition. */
10194	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10195	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10196	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10197	}
10198
10199
10200	/**
10201	* Advances the guest RIP after reading it from the VMCS.
10202	*
10203	* @returns VBox status code, no informational status codes.
10204	* @param pVCpu The cross context virtual CPU structure.
10205	* @param pVmxTransient Pointer to the VMX transient structure.
10206	*
10207	* @remarks No-long-jump zone!!!
10208	*/
10209	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10210	{
10211	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10212	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
10213	AssertRCReturn(rc, rc);
10214
10215	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10216
10217	/*
10218	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10219	* pending debug exception field as it takes care of priority of events.
10220	*
10221	* See Intel spec. 32.2.1 "Debug Exceptions".
10222	*/
10223	if ( !pVCpu->hm.s.fSingleInstruction
10224	&& pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10225	{
10226	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10227	AssertRCReturn(rc, rc);
10228	}
10229
10230	return VINF_SUCCESS;
10231	}
10232
10233
10234	/**
10235	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10236	* and update error record fields accordingly.
10237	*
10238	* @return VMX_IGS_* return codes.
10239	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10240	* wrong with the guest state.
10241	*
10242	* @param pVCpu The cross context virtual CPU structure.
10243	*
10244	* @remarks This function assumes our cache of the VMCS controls
10245	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10246	*/
10247	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10248	{
10249	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10250	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10251	uError = (err); \
10252	break; \
10253	} else do { } while (0)
10254
10255	int rc;
10256	PVM pVM = pVCpu->CTX_SUFF(pVM);
10257	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10258	uint32_t uError = VMX_IGS_ERROR;
10259	uint32_t u32Val;
10260	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10261
10262	do
10263	{
10264	/*
10265	* CR0.
10266	*/
10267	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10268	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10269	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10270	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10271	if (fUnrestrictedGuest)
10272	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10273
10274	uint32_t u32GuestCr0;
10275	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10276	AssertRCBreak(rc);
10277	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10278	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10279	if ( !fUnrestrictedGuest
10280	&& (u32GuestCr0 & X86_CR0_PG)
10281	&& !(u32GuestCr0 & X86_CR0_PE))
10282	{
10283	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10284	}
10285
10286	/*
10287	* CR4.
10288	*/
10289	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10290	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10291
10292	uint32_t u32GuestCr4;
10293	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10294	AssertRCBreak(rc);
10295	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10296	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10297
10298	/*
10299	* IA32_DEBUGCTL MSR.
10300	*/
10301	uint64_t u64Val;
10302	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10303	AssertRCBreak(rc);
10304	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10305	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10306	{
10307	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10308	}
10309	uint64_t u64DebugCtlMsr = u64Val;
10310
10311	#ifdef VBOX_STRICT
10312	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10313	AssertRCBreak(rc);
10314	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10315	#endif
10316	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10317
10318	/*
10319	* RIP and RFLAGS.
10320	*/
10321	uint32_t u32Eflags;
10322	#if HC_ARCH_BITS == 64
10323	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10324	AssertRCBreak(rc);
10325	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10326	if ( !fLongModeGuest
10327	\|\| !pCtx->cs.Attr.n.u1Long)
10328	{
10329	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10330	}
10331	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10332	* must be identical if the "IA-32e mode guest" VM-entry
10333	* control is 1 and CS.L is 1. No check applies if the
10334	* CPU supports 64 linear-address bits. */
10335
10336	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10337	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10338	AssertRCBreak(rc);
10339	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10340	VMX_IGS_RFLAGS_RESERVED);
10341	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10342	u32Eflags = u64Val;
10343	#else
10344	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10345	AssertRCBreak(rc);
10346	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10347	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10348	#endif
10349
10350	if ( fLongModeGuest
10351	\|\| ( fUnrestrictedGuest
10352	&& !(u32GuestCr0 & X86_CR0_PE)))
10353	{
10354	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10355	}
10356
10357	uint32_t u32EntryInfo;
10358	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10359	AssertRCBreak(rc);
10360	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10361	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10362	{
10363	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10364	}
10365
10366	/*
10367	* 64-bit checks.
10368	*/
10369	#if HC_ARCH_BITS == 64
10370	if (fLongModeGuest)
10371	{
10372	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10373	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10374	}
10375
10376	if ( !fLongModeGuest
10377	&& (u32GuestCr4 & X86_CR4_PCIDE))
10378	{
10379	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10380	}
10381
10382	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10383	* 51:32 beyond the processor's physical-address width are 0. */
10384
10385	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10386	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10387	{
10388	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10389	}
10390
10391	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10392	AssertRCBreak(rc);
10393	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10394
10395	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10396	AssertRCBreak(rc);
10397	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10398	#endif
10399
10400	/*
10401	* PERF_GLOBAL MSR.
10402	*/
10403	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10404	{
10405	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10406	AssertRCBreak(rc);
10407	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10408	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10409	}
10410
10411	/*
10412	* PAT MSR.
10413	*/
10414	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10415	{
10416	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10417	AssertRCBreak(rc);
10418	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10419	for (unsigned i = 0; i < 8; i++)
10420	{
10421	uint8_t u8Val = (u64Val & 0xff);
10422	if ( u8Val != 0 /* UC */
10423	&& u8Val != 1 /* WC */
10424	&& u8Val != 4 /* WT */
10425	&& u8Val != 5 /* WP */
10426	&& u8Val != 6 /* WB */
10427	&& u8Val != 7 /* UC- */)
10428	{
10429	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10430	}
10431	u64Val >>= 8;
10432	}
10433	}
10434
10435	/*
10436	* EFER MSR.
10437	*/
10438	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10439	{
10440	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10441	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10442	AssertRCBreak(rc);
10443	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10444	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10445	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10446	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10447	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10448	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10449	\|\| !(u32GuestCr0 & X86_CR0_PG)
10450	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10451	VMX_IGS_EFER_LMA_LME_MISMATCH);
10452	}
10453
10454	/*
10455	* Segment registers.
10456	*/
10457	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10458	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10459	if (!(u32Eflags & X86_EFL_VM))
10460	{
10461	/* CS */
10462	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10463	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10464	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10465	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10466	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10467	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10468	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10469	/* CS cannot be loaded with NULL in protected mode. */
10470	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10471	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10472	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10473	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10474	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10475	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10476	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10477	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10478	else
10479	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10480
10481	/* SS */
10482	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10483	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10484	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10485	if ( !(pCtx->cr0 & X86_CR0_PE)
10486	\|\| pCtx->cs.Attr.n.u4Type == 3)
10487	{
10488	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10489	}
10490	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10491	{
10492	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10493	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10494	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10495	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10496	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10497	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10498	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10499	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10500	}
10501
10502	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10503	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10504	{
10505	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10506	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10507	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10508	\|\| pCtx->ds.Attr.n.u4Type > 11
10509	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10510	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10511	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10512	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10513	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10514	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10515	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10516	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10517	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10518	}
10519	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10520	{
10521	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10522	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10523	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10524	\|\| pCtx->es.Attr.n.u4Type > 11
10525	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10526	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10527	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10528	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10529	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10530	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10531	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10532	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10533	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10534	}
10535	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10536	{
10537	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10538	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10539	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10540	\|\| pCtx->fs.Attr.n.u4Type > 11
10541	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10542	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10543	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10544	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10545	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10546	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10547	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10548	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10549	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10550	}
10551	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10552	{
10553	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10554	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10555	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10556	\|\| pCtx->gs.Attr.n.u4Type > 11
10557	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10558	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10559	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10560	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10561	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10562	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10563	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10564	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10565	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10566	}
10567	/* 64-bit capable CPUs. */
10568	#if HC_ARCH_BITS == 64
10569	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10570	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10571	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10572	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10573	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10574	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10575	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10576	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10577	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10578	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10579	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10580	#endif
10581	}
10582	else
10583	{
10584	/* V86 mode checks. */
10585	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10586	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10587	{
10588	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10589	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10590	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10591	}
10592	else
10593	{
10594	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10595	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10596	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10597	}
10598
10599	/* CS */
10600	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10601	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10602	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10603	/* SS */
10604	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10605	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10606	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10607	/* DS */
10608	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10609	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10610	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10611	/* ES */
10612	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10613	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10614	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10615	/* FS */
10616	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10617	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10618	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10619	/* GS */
10620	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10621	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10622	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10623	/* 64-bit capable CPUs. */
10624	#if HC_ARCH_BITS == 64
10625	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10626	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10627	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10628	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10629	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10630	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10631	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10632	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10633	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10634	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10635	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10636	#endif
10637	}
10638
10639	/*
10640	* TR.
10641	*/
10642	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10643	/* 64-bit capable CPUs. */
10644	#if HC_ARCH_BITS == 64
10645	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10646	#endif
10647	if (fLongModeGuest)
10648	{
10649	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10650	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10651	}
10652	else
10653	{
10654	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10655	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10656	VMX_IGS_TR_ATTR_TYPE_INVALID);
10657	}
10658	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10659	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10660	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10661	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10662	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10663	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10664	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10665	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10666
10667	/*
10668	* GDTR and IDTR.
10669	*/
10670	#if HC_ARCH_BITS == 64
10671	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10672	AssertRCBreak(rc);
10673	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10674
10675	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10676	AssertRCBreak(rc);
10677	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10678	#endif
10679
10680	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10681	AssertRCBreak(rc);
10682	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10683
10684	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10685	AssertRCBreak(rc);
10686	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10687
10688	/*
10689	* Guest Non-Register State.
10690	*/
10691	/* Activity State. */
10692	uint32_t u32ActivityState;
10693	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10694	AssertRCBreak(rc);
10695	HMVMX_CHECK_BREAK( !u32ActivityState
10696	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10697	VMX_IGS_ACTIVITY_STATE_INVALID);
10698	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10699	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10700	uint32_t u32IntrState;
10701	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10702	AssertRCBreak(rc);
10703	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10704	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10705	{
10706	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10707	}
10708
10709	/** @todo Activity state and injecting interrupts. Left as a todo since we
10710	* currently don't use activity states but ACTIVE. */
10711
10712	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10713	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10714
10715	/* Guest interruptibility-state. */
10716	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10717	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10718	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10719	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10720	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10721	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10722	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10723	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10724	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10725	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10726	{
10727	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10728	{
10729	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10730	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10731	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10732	}
10733	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10734	{
10735	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10736	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10737	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10738	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10739	}
10740	}
10741	/** @todo Assumes the processor is not in SMM. */
10742	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10743	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10744	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10745	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10746	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10747	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10748	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10749	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10750	{
10751	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10752	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10753	}
10754
10755	/* Pending debug exceptions. */
10756	#if HC_ARCH_BITS == 64
10757	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10758	AssertRCBreak(rc);
10759	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10760	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10761	u32Val = u64Val; /* For pending debug exceptions checks below. */
10762	#else
10763	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10764	AssertRCBreak(rc);
10765	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10766	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10767	#endif
10768
10769	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10770	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10771	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10772	{
10773	if ( (u32Eflags & X86_EFL_TF)
10774	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10775	{
10776	/* Bit 14 is PendingDebug.BS. */
10777	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10778	}
10779	if ( !(u32Eflags & X86_EFL_TF)
10780	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10781	{
10782	/* Bit 14 is PendingDebug.BS. */
10783	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10784	}
10785	}
10786
10787	/* VMCS link pointer. */
10788	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10789	AssertRCBreak(rc);
10790	if (u64Val != UINT64_C(0xffffffffffffffff))
10791	{
10792	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10793	/** @todo Bits beyond the processor's physical-address width MBZ. */
10794	/** @todo 32-bit located in memory referenced by value of this field (as a
10795	* physical address) must contain the processor's VMCS revision ID. */
10796	/** @todo SMM checks. */
10797	}
10798
10799	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10800	* not using Nested Paging? */
10801	if ( pVM->hm.s.fNestedPaging
10802	&& !fLongModeGuest
10803	&& CPUMIsGuestInPAEModeEx(pCtx))
10804	{
10805	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10806	AssertRCBreak(rc);
10807	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10808
10809	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10810	AssertRCBreak(rc);
10811	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10812
10813	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10814	AssertRCBreak(rc);
10815	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10816
10817	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10818	AssertRCBreak(rc);
10819	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10820	}
10821
10822	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10823	if (uError == VMX_IGS_ERROR)
10824	uError = VMX_IGS_REASON_NOT_FOUND;
10825	} while (0);
10826
10827	pVCpu->hm.s.u32HMError = uError;
10828	return uError;
10829
10830	#undef HMVMX_ERROR_BREAK
10831	#undef HMVMX_CHECK_BREAK
10832	}
10833
10834	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10835	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10836	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10837
10838	/** @name VM-exit handlers.
10839	* @{
10840	*/
10841
10842	/**
10843	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10844	*/
10845	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10846	{
10847	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10848	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10849	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10850	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10851	return VINF_SUCCESS;
10852	return VINF_EM_RAW_INTERRUPT;
10853	}
10854
10855
10856	/**
10857	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10858	*/
10859	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10860	{
10861	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10862	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10863
10864	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10865	AssertRCReturn(rc, rc);
10866
10867	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10868	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10869	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10870	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10871
10872	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10873	{
10874	/*
10875	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10876	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
10877	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10878	*
10879	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10880	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10881	*/
10882	VMXDispatchHostNmi();
10883	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10884	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10885	return VINF_SUCCESS;
10886	}
10887
10888	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10889	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10890	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10891	{ /* likely */ }
10892	else
10893	{
10894	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10895	rcStrictRc1 = VINF_SUCCESS;
10896	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10897	return rcStrictRc1;
10898	}
10899
10900	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10901	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10902	switch (uIntType)
10903	{
10904	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10905	Assert(uVector == X86_XCPT_DB);
10906	RT_FALL_THRU();
10907	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10908	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10909	RT_FALL_THRU();
10910	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10911	{
10912	/*
10913	* If there's any exception caused as a result of event injection, the resulting
10914	* secondary/final execption will be pending, we shall continue guest execution
10915	* after injecting the event. The page-fault case is complicated and we manually
10916	* handle any currently pending event in hmR0VmxExitXcptPF.
10917	*/
10918	if (!pVCpu->hm.s.Event.fPending)
10919	{ /* likely */ }
10920	else if (uVector != X86_XCPT_PF)
10921	{
10922	rc = VINF_SUCCESS;
10923	break;
10924	}
10925
10926	switch (uVector)
10927	{
10928	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
10929	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
10930	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
10931	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
10932	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
10933	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
10934
10935	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10936	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10937	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10938	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10939	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10940	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10941	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10942	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10943	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10944	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10945	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10946	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10947	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10948	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10949	default:
10950	{
10951	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10952	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10953	{
10954	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10955	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10956	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
10957
10958	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
10959	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10960	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10961	AssertRCReturn(rc, rc);
10962	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10963	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10964	0 /* GCPtrFaultAddress */);
10965	}
10966	else
10967	{
10968	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10969	pVCpu->hm.s.u32HMError = uVector;
10970	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10971	}
10972	break;
10973	}
10974	}
10975	break;
10976	}
10977
10978	default:
10979	{
10980	pVCpu->hm.s.u32HMError = uExitIntInfo;
10981	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10982	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10983	break;
10984	}
10985	}
10986	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10987	return rc;
10988	}
10989
10990
10991	/**
10992	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10993	*/
10994	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10995	{
10996	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10997
10998	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10999	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11000
11001	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11002	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11003	return VINF_SUCCESS;
11004	}
11005
11006
11007	/**
11008	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11009	*/
11010	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11011	{
11012	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11013	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11014	{
11015	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11016	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11017	}
11018
11019	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11020
11021	/*
11022	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11023	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11024	*/
11025	uint32_t fIntrState = 0;
11026	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11027	AssertRCReturn(rc, rc);
11028
11029	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11030	if ( fBlockSti
11031	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11032	{
11033	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11034	}
11035
11036	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11037	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11038
11039	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11040	return VINF_SUCCESS;
11041	}
11042
11043
11044	/**
11045	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11046	*/
11047	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11048	{
11049	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11050	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11051	}
11052
11053
11054	/**
11055	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11056	*/
11057	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11058	{
11059	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11060	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11061	}
11062
11063
11064	/**
11065	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11066	*/
11067	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11068	{
11069	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11070
11071	/*
11072	* Get the state we need and update the exit history entry.
11073	*/
11074	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11075	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
11076	AssertRCReturn(rc, rc);
11077
11078	VBOXSTRICTRC rcStrict;
11079	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11080	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
11081	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11082	if (!pExitRec)
11083	{
11084	/*
11085	* Regular CPUID instruction execution.
11086	*/
11087	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11088	if (rcStrict == VINF_SUCCESS)
11089	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RAX
11090	\| HM_CHANGED_GUEST_RCX \| HM_CHANGED_GUEST_RDX \| HM_CHANGED_GUEST_RBX);
11091	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11092	{
11093	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11094	rcStrict = VINF_SUCCESS;
11095	}
11096	}
11097	else
11098	{
11099	/*
11100	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
11101	*/
11102	int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11103	AssertRCReturn(rc2, rc2);
11104
11105	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11106	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11107
11108	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11109	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11110
11111	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11112	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11113	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11114	}
11115	return rcStrict;
11116	}
11117
11118
11119	/**
11120	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11121	*/
11122	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11123	{
11124	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11125	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11126	AssertRCReturn(rc, rc);
11127
11128	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11129	return VINF_EM_RAW_EMULATE_INSTR;
11130
11131	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11132	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11133	}
11134
11135
11136	/**
11137	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11138	*/
11139	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11140	{
11141	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11142	int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11143	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11144	AssertRCReturn(rc, rc);
11145
11146	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11147	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11148	{
11149	/* If we get a spurious VM-exit when offsetting is enabled,
11150	we must reset offsetting on VM-reentry. See @bugref{6634}. */
11151	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11152	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11153	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
11154	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
11155	}
11156	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11157	{
11158	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11159	rcStrict = VINF_SUCCESS;
11160	}
11161	return rcStrict;
11162	}
11163
11164
11165	/**
11166	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11167	*/
11168	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11169	{
11170	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11171	int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
11172	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11173	AssertRCReturn(rc, rc);
11174
11175	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11176	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11177	{
11178	/* If we get a spurious VM-exit when offsetting is enabled,
11179	we must reset offsetting on VM-reentry. See @bugref{6634}. */
11180	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11181	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11182	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
11183	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX \| HM_CHANGED_GUEST_RCX);
11184	}
11185	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11186	{
11187	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11188	rcStrict = VINF_SUCCESS;
11189	}
11190	return rcStrict;
11191	}
11192
11193
11194	/**
11195	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11196	*/
11197	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11198	{
11199	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11200	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
11201	AssertRCReturn(rc, rc);
11202
11203	PVM pVM = pVCpu->CTX_SUFF(pVM);
11204	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11205	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11206	if (RT_LIKELY(rc == VINF_SUCCESS))
11207	{
11208	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11209	Assert(pVmxTransient->cbInstr == 2);
11210	}
11211	else
11212	{
11213	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11214	rc = VERR_EM_INTERPRETER;
11215	}
11216	return rc;
11217	}
11218
11219
11220	/**
11221	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11222	*/
11223	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11224	{
11225	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11226
11227	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11228	if (EMAreHypercallInstructionsEnabled(pVCpu))
11229	{
11230	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_SS
11231	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
11232	AssertRCReturn(rc, rc);
11233
11234	/* Perform the hypercall. */
11235	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11236	if (rcStrict == VINF_SUCCESS)
11237	{
11238	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11239	AssertRCReturn(rc, rc);
11240	}
11241	else
11242	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11243	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11244	\|\| RT_FAILURE(rcStrict));
11245
11246	/* If the hypercall changes anything other than guest's general-purpose registers,
11247	we would need to reload the guest changed bits here before VM-entry. */
11248	}
11249	else
11250	Log4Func(("Hypercalls not enabled\n"));
11251
11252	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11253	if (RT_FAILURE(rcStrict))
11254	{
11255	hmR0VmxSetPendingXcptUD(pVCpu);
11256	rcStrict = VINF_SUCCESS;
11257	}
11258
11259	return rcStrict;
11260	}
11261
11262
11263	/**
11264	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11265	*/
11266	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11267	{
11268	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11269	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11270
11271	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11272	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11273	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
11274	AssertRCReturn(rc, rc);
11275
11276	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQualification);
11277
11278	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
11279	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11280	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11281	{
11282	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11283	rcStrict = VINF_SUCCESS;
11284	}
11285	else
11286	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n",
11287	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11288	return rcStrict;
11289	}
11290
11291
11292	/**
11293	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11294	*/
11295	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11296	{
11297	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11298	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
11299	AssertRCReturn(rc, rc);
11300
11301	PVM pVM = pVCpu->CTX_SUFF(pVM);
11302	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11303	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11304	if (RT_LIKELY(rc == VINF_SUCCESS))
11305	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11306	else
11307	{
11308	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11309	rc = VERR_EM_INTERPRETER;
11310	}
11311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11312	return rc;
11313	}
11314
11315
11316	/**
11317	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11318	*/
11319	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11320	{
11321	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11322	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
11323	AssertRCReturn(rc, rc);
11324
11325	PVM pVM = pVCpu->CTX_SUFF(pVM);
11326	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11327	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11328	rc = VBOXSTRICTRC_VAL(rc2);
11329	if (RT_LIKELY( rc == VINF_SUCCESS
11330	\|\| rc == VINF_EM_HALT))
11331	{
11332	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11333	AssertRCReturn(rc3, rc3);
11334
11335	if ( rc == VINF_EM_HALT
11336	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
11337	rc = VINF_SUCCESS;
11338	}
11339	else
11340	{
11341	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11342	rc = VERR_EM_INTERPRETER;
11343	}
11344	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11345	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11346	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11347	return rc;
11348	}
11349
11350
11351	/**
11352	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11353	*/
11354	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11355	{
11356	/*
11357	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11358	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11359	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11360	* VMX root operation. If we get here, something funny is going on.
11361	*
11362	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11363	*/
11364	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11365	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11366	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11367	}
11368
11369
11370	/**
11371	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11372	*/
11373	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11374	{
11375	/*
11376	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
11377	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11378	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11379	* an SMI. If we get here, something funny is going on.
11380	*
11381	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11382	* See Intel spec. 25.3 "Other Causes of VM-Exits"
11383	*/
11384	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11385	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11386	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11387	}
11388
11389
11390	/**
11391	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11392	*/
11393	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11394	{
11395	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11396	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11397	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11398	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11399	}
11400
11401
11402	/**
11403	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11404	*/
11405	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11406	{
11407	/*
11408	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11409	* We don't make use of it as our guests don't have direct access to the host LAPIC.
11410	* See Intel spec. 25.3 "Other Causes of VM-exits".
11411	*/
11412	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11413	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11414	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11415	}
11416
11417
11418	/**
11419	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11420	* VM-exit.
11421	*/
11422	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11423	{
11424	/*
11425	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11426	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11427	*
11428	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11429	* See Intel spec. "23.8 Restrictions on VMX operation".
11430	*/
11431	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11432	return VINF_SUCCESS;
11433	}
11434
11435
11436	/**
11437	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11438	* VM-exit.
11439	*/
11440	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11441	{
11442	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11443	return VINF_EM_RESET;
11444	}
11445
11446
11447	/**
11448	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11449	*/
11450	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11451	{
11452	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11453	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11454
11455	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11456	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11457	AssertRCReturn(rc, rc);
11458
11459	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11460	rc = VINF_SUCCESS;
11461	else
11462	rc = VINF_EM_HALT;
11463
11464	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11465	if (rc != VINF_SUCCESS)
11466	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11467	return rc;
11468	}
11469
11470
11471	/**
11472	* VM-exit handler for instructions that result in a \#UD exception delivered to
11473	* the guest.
11474	*/
11475	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11476	{
11477	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11478	hmR0VmxSetPendingXcptUD(pVCpu);
11479	return VINF_SUCCESS;
11480	}
11481
11482
11483	/**
11484	* VM-exit handler for expiry of the VMX preemption timer.
11485	*/
11486	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11487	{
11488	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11489
11490	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11491	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11492
11493	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11494	PVM pVM = pVCpu->CTX_SUFF(pVM);
11495	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11496	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11497	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11498	}
11499
11500
11501	/**
11502	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11503	*/
11504	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11505	{
11506	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11507
11508	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11509	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
11510	AssertRCReturn(rc, rc);
11511
11512	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11513	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
11514	: HM_CHANGED_XCPT_RAISED_MASK);
11515
11516	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11517	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11518
11519	return rcStrict;
11520	}
11521
11522
11523	/**
11524	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11525	*/
11526	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11527	{
11528	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11529	/** @todo Use VM-exit instruction information. */
11530	return VERR_EM_INTERPRETER;
11531	}
11532
11533
11534	/**
11535	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11536	* Error VM-exit.
11537	*/
11538	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11539	{
11540	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11541	AssertRCReturn(rc, rc);
11542	rc = hmR0VmxCheckVmcsCtls(pVCpu);
11543	if (RT_FAILURE(rc))
11544	return rc;
11545
11546	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11547	NOREF(uInvalidReason);
11548
11549	#ifdef VBOX_STRICT
11550	uint32_t fIntrState;
11551	RTHCUINTREG uHCReg;
11552	uint64_t u64Val;
11553	uint32_t u32Val;
11554
11555	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11556	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11557	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11558	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11559	AssertRCReturn(rc, rc);
11560
11561	Log4(("uInvalidReason %u\n", uInvalidReason));
11562	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11563	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11564	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11565	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11566
11567	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11568	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11569	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11570	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11571	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11572	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11573	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11574	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11575	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11576	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11577	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11578	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11579
11580	hmR0DumpRegs(pVCpu);
11581	#else
11582	NOREF(pVmxTransient);
11583	#endif
11584
11585	return VERR_VMX_INVALID_GUEST_STATE;
11586	}
11587
11588
11589	/**
11590	* VM-exit handler for VM-entry failure due to an MSR-load
11591	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11592	*/
11593	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11594	{
11595	AssertMsgFailed(("Unexpected MSR-load exit\n"));
11596	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11597	}
11598
11599
11600	/**
11601	* VM-exit handler for VM-entry failure due to a machine-check event
11602	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11603	*/
11604	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11605	{
11606	AssertMsgFailed(("Unexpected machine-check event exit\n"));
11607	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11608	}
11609
11610
11611	/**
11612	* VM-exit handler for all undefined reasons. Should never ever happen.. in
11613	* theory.
11614	*/
11615	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11616	{
11617	RT_NOREF2(pVCpu, pVmxTransient);
11618	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11619	return VERR_VMX_UNDEFINED_EXIT_CODE;
11620	}
11621
11622
11623	/**
11624	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11625	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11626	* Conditional VM-exit.
11627	*/
11628	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11629	{
11630	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11631
11632	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11633	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11634	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11635	return VERR_EM_INTERPRETER;
11636	AssertMsgFailed(("Unexpected XDTR access\n"));
11637	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11638	}
11639
11640
11641	/**
11642	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11643	*/
11644	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11645	{
11646	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11647
11648	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11649	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11650	return VERR_EM_INTERPRETER;
11651	AssertMsgFailed(("Unexpected RDRAND exit\n"));
11652	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11653	}
11654
11655
11656	/**
11657	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11658	*/
11659	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11660	{
11661	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11662
11663	/** @todo Optimize this: We currently drag in in the whole MSR state
11664	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11665	* MSRs required. That would require changes to IEM and possibly CPUM too.
11666	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11667	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11668	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11669	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
11670	AssertRCReturn(rc, rc);
11671
11672	Log4Func(("ecx=%#RX32\n", idMsr));
11673
11674	#ifdef VBOX_STRICT
11675	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11676	{
11677	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11678	&& idMsr != MSR_K6_EFER)
11679	{
11680	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11681	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11682	}
11683	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11684	{
11685	VMXMSREXITREAD enmRead;
11686	VMXMSREXITWRITE enmWrite;
11687	int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11688	AssertRCReturn(rc2, rc2);
11689	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11690	{
11691	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11692	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11693	}
11694	}
11695	}
11696	#endif
11697
11698	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11699	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11700	if (rcStrict == VINF_SUCCESS)
11701	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
11702	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
11703	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11704	{
11705	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11706	rcStrict = VINF_SUCCESS;
11707	}
11708	else
11709	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11710
11711	return rcStrict;
11712	}
11713
11714
11715	/**
11716	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11717	*/
11718	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11719	{
11720	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11721
11722	/** @todo Optimize this: We currently drag in in the whole MSR state
11723	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11724	* MSRs required. That would require changes to IEM and possibly CPUM too.
11725	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11726	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11727	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11728	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
11729	AssertRCReturn(rc, rc);
11730
11731	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11732
11733	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11734	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11735
11736	if (rcStrict == VINF_SUCCESS)
11737	{
11738	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11739
11740	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
11741	if ( idMsr == MSR_IA32_APICBASE
11742	\|\| ( idMsr >= MSR_IA32_X2APIC_START
11743	&& idMsr <= MSR_IA32_X2APIC_END))
11744	{
11745	/*
11746	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11747	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11748	*/
11749	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11750	}
11751	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11752	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11753	else if (idMsr == MSR_K6_EFER)
11754	{
11755	/*
11756	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11757	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11758	* the other bits as well, SCE and NXE. See @bugref{7368}.
11759	*/
11760	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS
11761	\| HM_CHANGED_VMX_EXIT_CTLS);
11762	}
11763
11764	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11765	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11766	{
11767	switch (idMsr)
11768	{
11769	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11770	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11771	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11772	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11773	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11774	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11775	default:
11776	{
11777	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11778	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11779	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11780	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11781	break;
11782	}
11783	}
11784	}
11785	#ifdef VBOX_STRICT
11786	else
11787	{
11788	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11789	switch (idMsr)
11790	{
11791	case MSR_IA32_SYSENTER_CS:
11792	case MSR_IA32_SYSENTER_EIP:
11793	case MSR_IA32_SYSENTER_ESP:
11794	case MSR_K8_FS_BASE:
11795	case MSR_K8_GS_BASE:
11796	{
11797	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
11798	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11799	}
11800
11801	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11802	default:
11803	{
11804	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11805	{
11806	/* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11807	if (idMsr != MSR_K6_EFER)
11808	{
11809	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11810	idMsr));
11811	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11812	}
11813	}
11814
11815	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11816	{
11817	VMXMSREXITREAD enmRead;
11818	VMXMSREXITWRITE enmWrite;
11819	int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11820	AssertRCReturn(rc2, rc2);
11821	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11822	{
11823	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
11824	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11825	}
11826	}
11827	break;
11828	}
11829	}
11830	}
11831	#endif /* VBOX_STRICT */
11832	}
11833	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11834	{
11835	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11836	rcStrict = VINF_SUCCESS;
11837	}
11838	else
11839	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11840
11841	return rcStrict;
11842	}
11843
11844
11845	/**
11846	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11847	*/
11848	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11849	{
11850	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11851	/** @todo The guest has likely hit a contended spinlock. We might want to
11852	* poke a schedule different guest VCPU. */
11853	return VINF_EM_RAW_INTERRUPT;
11854	}
11855
11856
11857	/**
11858	* VM-exit handler for when the TPR value is lowered below the specified
11859	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11860	*/
11861	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11862	{
11863	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11864	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11865
11866	/*
11867	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11868	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11869	*/
11870	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11871	return VINF_SUCCESS;
11872	}
11873
11874
11875	/**
11876	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11877	* VM-exit.
11878	*
11879	* @retval VINF_SUCCESS when guest execution can continue.
11880	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11881	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11882	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11883	* interpreter.
11884	*/
11885	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11886	{
11887	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11888	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11889
11890	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11891	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11892	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11893	AssertRCReturn(rc, rc);
11894
11895	VBOXSTRICTRC rcStrict;
11896	PVM pVM = pVCpu->CTX_SUFF(pVM);
11897	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11898	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11899	switch (uAccessType)
11900	{
11901	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11902	{
11903	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11904	VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11905	VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11906	AssertMsg( rcStrict == VINF_SUCCESS
11907	\|\| rcStrict == VINF_IEM_RAISED_XCPT
11908	\|\| rcStrict == VINF_PGM_CHANGE_MODE
11909	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11910
11911	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11912	{
11913	case 0:
11914	{
11915	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11916	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
11917	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11918	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
11919	break;
11920	}
11921
11922	case 2:
11923	{
11924	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11925	/* Nothing to do here, CR2 it's not part of the VMCS. */
11926	break;
11927	}
11928
11929	case 3:
11930	{
11931	Assert( !pVM->hm.s.fNestedPaging
11932	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11933	\|\| pVCpu->hm.s.fUsingDebugLoop);
11934	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11935	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11936	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
11937	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
11938	break;
11939	}
11940
11941	case 4:
11942	{
11943	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11944	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11945	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
11946	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11947	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11948	break;
11949	}
11950
11951	case 8:
11952	{
11953	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11954	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11955	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11956	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
11957	break;
11958	}
11959	default:
11960	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11961	break;
11962	}
11963	break;
11964	}
11965
11966	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
11967	{
11968	Assert( !pVM->hm.s.fNestedPaging
11969	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11970	\|\| pVCpu->hm.s.fUsingDebugLoop
11971	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
11972	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11973	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
11974	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11975
11976	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
11977	VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
11978	VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
11979	AssertMsg( rcStrict == VINF_SUCCESS
11980	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11981	#ifdef VBOX_WITH_STATISTICS
11982	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11983	{
11984	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
11985	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
11986	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
11987	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
11988	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
11989	}
11990	#endif
11991	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11992	VBOXSTRICTRC_VAL(rcStrict)));
11993	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
11994	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
11995	else
11996	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11997	break;
11998	}
11999
12000	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12001	{
12002	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12003	AssertMsg( rcStrict == VINF_SUCCESS
12004	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12005
12006	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
12007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12008	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12009	break;
12010	}
12011
12012	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12013	{
12014	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12015	VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12016	AssertMsg( rcStrict == VINF_SUCCESS
12017	\|\| rcStrict == VINF_IEM_RAISED_XCPT
12018	\|\| rcStrict == VINF_PGM_CHANGE_MODE,
12019	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12020
12021	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
12022	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12023	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12024	break;
12025	}
12026
12027	default:
12028	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12029	VERR_VMX_UNEXPECTED_EXCEPTION);
12030	}
12031
12032	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
12033	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
12034	if (rcStrict == VINF_IEM_RAISED_XCPT)
12035	{
12036	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
12037	rcStrict = VINF_SUCCESS;
12038	}
12039
12040	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12041	NOREF(pVM);
12042	return rcStrict;
12043	}
12044
12045
12046	/**
12047	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12048	* VM-exit.
12049	*/
12050	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12051	{
12052	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12053	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12054
12055	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12056	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12057	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12058	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER);
12059	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12060	AssertRCReturn(rc, rc);
12061
12062	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12063	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12064	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12065	bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12066	== VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12067	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12068	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12069	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12070	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12071
12072	/*
12073	* Update exit history to see if this exit can be optimized.
12074	*/
12075	VBOXSTRICTRC rcStrict;
12076	PCEMEXITREC pExitRec = NULL;
12077	if ( !fGstStepping
12078	&& !fDbgStepping)
12079	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12080	!fIOString
12081	? !fIOWrite
12082	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12083	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12084	: !fIOWrite
12085	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12086	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12087	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12088	if (!pExitRec)
12089	{
12090	/* I/O operation lookup arrays. */
12091	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12092	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12093	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12094	uint32_t const cbInstr = pVmxTransient->cbInstr;
12095	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12096	PVM pVM = pVCpu->CTX_SUFF(pVM);
12097	if (fIOString)
12098	{
12099	/*
12100	* INS/OUTS - I/O String instruction.
12101	*
12102	* Use instruction-information if available, otherwise fall back on
12103	* interpreting the instruction.
12104	*/
12105	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12106	fIOWrite ? 'w' : 'r'));
12107	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12108	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12109	{
12110	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12111	AssertRCReturn(rc2, rc2);
12112	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12113	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12114	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12115	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12116	if (fIOWrite)
12117	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12118	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12119	else
12120	{
12121	/*
12122	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12123	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12124	* See Intel Instruction spec. for "INS".
12125	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12126	*/
12127	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12128	}
12129	}
12130	else
12131	rcStrict = IEMExecOne(pVCpu);
12132
12133	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12134	fUpdateRipAlready = true;
12135	}
12136	else
12137	{
12138	/*
12139	* IN/OUT - I/O instruction.
12140	*/
12141	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12142	fIOWrite ? 'w' : 'r'));
12143	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12144	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12145	if (fIOWrite)
12146	{
12147	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12148	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12149	}
12150	else
12151	{
12152	uint32_t u32Result = 0;
12153	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12154	if (IOM_SUCCESS(rcStrict))
12155	{
12156	/* Save result of I/O IN instr. in AL/AX/EAX. */
12157	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12158	}
12159	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12160	HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12161	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12162	}
12163	}
12164
12165	if (IOM_SUCCESS(rcStrict))
12166	{
12167	if (!fUpdateRipAlready)
12168	{
12169	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12170	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12171	}
12172
12173	/*
12174	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12175	* while booting Fedora 17 64-bit guest.
12176	*
12177	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12178	*/
12179	if (fIOString)
12180	{
12181	/** @todo Single-step for INS/OUTS with REP prefix? */
12182	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12183	}
12184	else if ( !fDbgStepping
12185	&& fGstStepping)
12186	{
12187	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12188	AssertRCReturn(rc, rc);
12189	}
12190
12191	/*
12192	* If any I/O breakpoints are armed, we need to check if one triggered
12193	* and take appropriate action.
12194	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12195	*/
12196	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12197	AssertRCReturn(rc, rc);
12198
12199	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12200	* execution engines about whether hyper BPs and such are pending. */
12201	uint32_t const uDr7 = pCtx->dr[7];
12202	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12203	&& X86_DR7_ANY_RW_IO(uDr7)
12204	&& (pCtx->cr4 & X86_CR4_DE))
12205	\|\| DBGFBpIsHwIoArmed(pVM)))
12206	{
12207	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12208
12209	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12210	VMMRZCallRing3Disable(pVCpu);
12211	HM_DISABLE_PREEMPT(pVCpu);
12212
12213	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12214
12215	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12216	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12217	{
12218	/* Raise #DB. */
12219	if (fIsGuestDbgActive)
12220	ASMSetDR6(pCtx->dr[6]);
12221	if (pCtx->dr[7] != uDr7)
12222	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
12223
12224	hmR0VmxSetPendingXcptDB(pVCpu);
12225	}
12226	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12227	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12228	else if ( rcStrict2 != VINF_SUCCESS
12229	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12230	rcStrict = rcStrict2;
12231	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12232
12233	HM_RESTORE_PREEMPT();
12234	VMMRZCallRing3Enable(pVCpu);
12235	}
12236	}
12237
12238	#ifdef VBOX_STRICT
12239	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12240	Assert(!fIOWrite);
12241	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12242	Assert(fIOWrite);
12243	else
12244	{
12245	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12246	* statuses, that the VMM device and some others may return. See
12247	* IOM_SUCCESS() for guidance. */
12248	AssertMsg( RT_FAILURE(rcStrict)
12249	\|\| rcStrict == VINF_SUCCESS
12250	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12251	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12252	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12253	\|\| rcStrict == VINF_EM_RAW_TO_R3
12254	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12255	# endif
12256	}
12257	#endif
12258	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12259	}
12260	else
12261	{
12262	/*
12263	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
12264	*/
12265	int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12266	AssertRCReturn(rc2, rc2);
12267	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12268	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12269	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12270	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12271	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12272	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12273
12274	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12275	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12276
12277	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12278	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12279	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12280	}
12281	return rcStrict;
12282	}
12283
12284
12285	/**
12286	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12287	* VM-exit.
12288	*/
12289	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12290	{
12291	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12292
12293	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12294	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12295	AssertRCReturn(rc, rc);
12296	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12297	{
12298	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12299	AssertRCReturn(rc, rc);
12300	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12301	{
12302	uint32_t uErrCode;
12303	RTGCUINTPTR GCPtrFaultAddress;
12304	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12305	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12306	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12307	if (fErrorCodeValid)
12308	{
12309	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12310	AssertRCReturn(rc, rc);
12311	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12312	}
12313	else
12314	uErrCode = 0;
12315
12316	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12317	&& uVector == X86_XCPT_PF)
12318	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12319	else
12320	GCPtrFaultAddress = 0;
12321
12322	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12323	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12324
12325	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12326	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12327	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12328	}
12329	}
12330
12331	/* Fall back to the interpreter to emulate the task-switch. */
12332	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12333	return VERR_EM_INTERPRETER;
12334	}
12335
12336
12337	/**
12338	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12339	*/
12340	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12341	{
12342	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12343	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12344	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12345	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12346	AssertRCReturn(rc, rc);
12347	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12348	return VINF_EM_DBG_STEPPED;
12349	}
12350
12351
12352	/**
12353	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12354	*/
12355	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12356	{
12357	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12358
12359	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12360
12361	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12362	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12363	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12364	{
12365	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12366	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12367	{
12368	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12369	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12370	}
12371	}
12372	else
12373	{
12374	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12375	rcStrict1 = VINF_SUCCESS;
12376	return rcStrict1;
12377	}
12378
12379	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12380	int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12381	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12382	AssertRCReturn(rc, rc);
12383
12384	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12385	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12386	VBOXSTRICTRC rcStrict2;
12387	switch (uAccessType)
12388	{
12389	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12390	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12391	{
12392	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12393	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12394	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12395
12396	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12397	GCPhys &= PAGE_BASE_GC_MASK;
12398	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12399	PVM pVM = pVCpu->CTX_SUFF(pVM);
12400	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12401	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12402
12403	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12404	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12405	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12406	CPUMCTX2CORE(pCtx), GCPhys);
12407	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12408	if ( rcStrict2 == VINF_SUCCESS
12409	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12410	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12411	{
12412	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
12413	\| HM_CHANGED_GUEST_APIC_TPR);
12414	rcStrict2 = VINF_SUCCESS;
12415	}
12416	break;
12417	}
12418
12419	default:
12420	Log4Func(("uAccessType=%#x\n", uAccessType));
12421	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12422	break;
12423	}
12424
12425	if (rcStrict2 != VINF_SUCCESS)
12426	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12427	return rcStrict2;
12428	}
12429
12430
12431	/**
12432	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12433	* VM-exit.
12434	*/
12435	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12436	{
12437	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12438
12439	/* We should -not- get this VM-exit if the guest's debug registers were active. */
12440	if (pVmxTransient->fWasGuestDebugStateActive)
12441	{
12442	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12443	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12444	}
12445
12446	if ( !pVCpu->hm.s.fSingleInstruction
12447	&& !pVmxTransient->fWasHyperDebugStateActive)
12448	{
12449	Assert(!DBGFIsStepping(pVCpu));
12450	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12451
12452	/* Don't intercept MOV DRx any more. */
12453	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12454	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12455	AssertRCReturn(rc, rc);
12456
12457	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12458	VMMRZCallRing3Disable(pVCpu);
12459	HM_DISABLE_PREEMPT(pVCpu);
12460
12461	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12462	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12463	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
12464
12465	HM_RESTORE_PREEMPT();
12466	VMMRZCallRing3Enable(pVCpu);
12467
12468	#ifdef VBOX_WITH_STATISTICS
12469	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12470	AssertRCReturn(rc, rc);
12471	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12472	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12473	else
12474	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12475	#endif
12476	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12477	return VINF_SUCCESS;
12478	}
12479
12480	/*
12481	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12482	* Update the segment registers and DR7 from the CPU.
12483	*/
12484	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12485	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12486	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
12487	AssertRCReturn(rc, rc);
12488	Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12489
12490	PVM pVM = pVCpu->CTX_SUFF(pVM);
12491	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12492	{
12493	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12494	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12495	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12496	if (RT_SUCCESS(rc))
12497	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12498	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12499	}
12500	else
12501	{
12502	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12503	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12504	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12505	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12506	}
12507
12508	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
12509	if (RT_SUCCESS(rc))
12510	{
12511	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12512	AssertRCReturn(rc2, rc2);
12513	return VINF_SUCCESS;
12514	}
12515	return rc;
12516	}
12517
12518
12519	/**
12520	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12521	* Conditional VM-exit.
12522	*/
12523	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12524	{
12525	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12526	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12527
12528	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12529	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12530	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12531	{
12532	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12533	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12534	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12535	{
12536	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12537	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12538	}
12539	}
12540	else
12541	{
12542	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12543	rcStrict1 = VINF_SUCCESS;
12544	return rcStrict1;
12545	}
12546
12547	/*
12548	* Get sufficent state and update the exit history entry.
12549	*/
12550	RTGCPHYS GCPhys;
12551	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12552	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12553	AssertRCReturn(rc, rc);
12554
12555	VBOXSTRICTRC rcStrict;
12556	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12557	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
12558	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12559	if (!pExitRec)
12560	{
12561	/*
12562	* If we succeed, resume guest execution.
12563	* If we fail in interpreting the instruction because we couldn't get the guest physical address
12564	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12565	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12566	* weird case. See @bugref{6043}.
12567	*/
12568	PVM pVM = pVCpu->CTX_SUFF(pVM);
12569	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12570	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12571	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12572	if ( rcStrict == VINF_SUCCESS
12573	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12574	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
12575	{
12576	/* Successfully handled MMIO operation. */
12577	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
12578	\| HM_CHANGED_GUEST_APIC_TPR);
12579	rcStrict = VINF_SUCCESS;
12580	}
12581	}
12582	else
12583	{
12584	/*
12585	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
12586	*/
12587	int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12588	AssertRCReturn(rc2, rc2);
12589
12590	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12591	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12592
12593	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12594	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12595
12596	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12597	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12598	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12599	}
12600	return VBOXSTRICTRC_TODO(rcStrict);
12601	}
12602
12603
12604	/**
12605	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12606	* VM-exit.
12607	*/
12608	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12609	{
12610	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12611	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12612
12613	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12614	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12615	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12616	{
12617	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12618	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12619	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12620	}
12621	else
12622	{
12623	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12624	rcStrict1 = VINF_SUCCESS;
12625	return rcStrict1;
12626	}
12627
12628	RTGCPHYS GCPhys;
12629	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12630	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12631	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12632	AssertRCReturn(rc, rc);
12633
12634	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12635	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12636
12637	RTGCUINT uErrorCode = 0;
12638	if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12639	uErrorCode \|= X86_TRAP_PF_ID;
12640	if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12641	uErrorCode \|= X86_TRAP_PF_RW;
12642	if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12643	uErrorCode \|= X86_TRAP_PF_P;
12644
12645	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12646
12647
12648	/* Handle the pagefault trap for the nested shadow table. */
12649	PVM pVM = pVCpu->CTX_SUFF(pVM);
12650	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12651
12652	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12653	uErrorCode, pCtx->cs.Sel, pCtx->rip));
12654
12655	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12656	TRPMResetTrap(pVCpu);
12657
12658	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12659	if ( rcStrict2 == VINF_SUCCESS
12660	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12661	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12662	{
12663	/* Successfully synced our nested page tables. */
12664	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12665	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
12666	return VINF_SUCCESS;
12667	}
12668
12669	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12670	return rcStrict2;
12671	}
12672
12673	/** @} */
12674
12675	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12676	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12677	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12678
12679	/** @name VM-exit exception handlers.
12680	* @{
12681	*/
12682
12683	/**
12684	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
12685	*/
12686	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12687	{
12688	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12689	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12690
12691	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12692	AssertRCReturn(rc, rc);
12693
12694	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12695	{
12696	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12697	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12698
12699	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12700	* provides VM-exit instruction length. If this causes problem later,
12701	* disassemble the instruction like it's done on AMD-V. */
12702	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12703	AssertRCReturn(rc2, rc2);
12704	return rc;
12705	}
12706
12707	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12708	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12709	return rc;
12710	}
12711
12712
12713	/**
12714	* VM-exit exception handler for \#BP (Breakpoint exception).
12715	*/
12716	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12717	{
12718	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12719	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12720
12721	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12722	AssertRCReturn(rc, rc);
12723
12724	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12725	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
12726	if (rc == VINF_EM_RAW_GUEST_TRAP)
12727	{
12728	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12729	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12730	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12731	AssertRCReturn(rc, rc);
12732
12733	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12734	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12735	}
12736
12737	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
12738	return rc;
12739	}
12740
12741
12742	/**
12743	* VM-exit exception handler for \#AC (alignment check exception).
12744	*/
12745	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12746	{
12747	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12748
12749	/*
12750	* Re-inject it. We'll detect any nesting before getting here.
12751	*/
12752	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12753	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12754	AssertRCReturn(rc, rc);
12755	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12756
12757	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12758	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12759	return VINF_SUCCESS;
12760	}
12761
12762
12763	/**
12764	* VM-exit exception handler for \#DB (Debug exception).
12765	*/
12766	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12767	{
12768	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12769	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12770
12771	/*
12772	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12773	* for processing.
12774	*/
12775	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12776
12777	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12778	uint64_t uDR6 = X86_DR6_INIT_VAL;
12779	uDR6 \|= ( pVmxTransient->uExitQualification
12780	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
12781
12782	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12783	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12784	Log6Func(("rc=%Rrc\n", rc));
12785	if (rc == VINF_EM_RAW_GUEST_TRAP)
12786	{
12787	/*
12788	* The exception was for the guest. Update DR6, DR7.GD and
12789	* IA32_DEBUGCTL.LBR before forwarding it.
12790	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12791	*/
12792	VMMRZCallRing3Disable(pVCpu);
12793	HM_DISABLE_PREEMPT(pVCpu);
12794
12795	pCtx->dr[6] &= ~X86_DR6_B_MASK;
12796	pCtx->dr[6] \|= uDR6;
12797	if (CPUMIsGuestDebugStateActive(pVCpu))
12798	ASMSetDR6(pCtx->dr[6]);
12799
12800	HM_RESTORE_PREEMPT();
12801	VMMRZCallRing3Enable(pVCpu);
12802
12803	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12804	AssertRCReturn(rc, rc);
12805
12806	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12807	pCtx->dr[7] &= ~X86_DR7_GD;
12808
12809	/* Paranoia. */
12810	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12811	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
12812
12813	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
12814	AssertRCReturn(rc, rc);
12815
12816	/*
12817	* Raise #DB in the guest.
12818	*
12819	* It is important to reflect exactly what the VM-exit gave us (preserving the
12820	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12821	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12822	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12823	*
12824	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12825	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12826	*/
12827	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12828	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12829	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12830	AssertRCReturn(rc, rc);
12831	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12832	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12833	return VINF_SUCCESS;
12834	}
12835
12836	/*
12837	* Not a guest trap, must be a hypervisor related debug event then.
12838	* Update DR6 in case someone is interested in it.
12839	*/
12840	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12841	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12842	CPUMSetHyperDR6(pVCpu, uDR6);
12843
12844	return rc;
12845	}
12846
12847	/**
12848	* VM-exit exception handler for \#GP (General-protection exception).
12849	*
12850	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12851	*/
12852	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12853	{
12854	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12855	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12856
12857	int rc;
12858	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12859	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12860	{ /* likely */ }
12861	else
12862	{
12863	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12864	Assert(pVCpu->hm.s.fUsingDebugLoop);
12865	#endif
12866	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12867	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12868	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12869	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12870	rc \|= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12871	AssertRCReturn(rc, rc);
12872	Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
12873	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
12874	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12875	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12876	return rc;
12877	}
12878
12879	Assert(CPUMIsGuestInRealModeEx(pCtx));
12880	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12881
12882	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12883	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12884	AssertRCReturn(rc, rc);
12885
12886	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12887	uint32_t cbOp = 0;
12888	PVM pVM = pVCpu->CTX_SUFF(pVM);
12889	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12890	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12891	if (RT_SUCCESS(rc))
12892	{
12893	rc = VINF_SUCCESS;
12894	Assert(cbOp == pDis->cbInstr);
12895	Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pCtx->cs.Sel, pCtx->rip));
12896	switch (pDis->pCurInstr->uOpcode)
12897	{
12898	case OP_CLI:
12899	{
12900	pCtx->eflags.Bits.u1IF = 0;
12901	pCtx->eflags.Bits.u1RF = 0;
12902	pCtx->rip += pDis->cbInstr;
12903	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
12904	if ( !fDbgStepping
12905	&& pCtx->eflags.Bits.u1TF)
12906	{
12907	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12908	AssertRCReturn(rc, rc);
12909	}
12910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12911	break;
12912	}
12913
12914	case OP_STI:
12915	{
12916	bool fOldIF = pCtx->eflags.Bits.u1IF;
12917	pCtx->eflags.Bits.u1IF = 1;
12918	pCtx->eflags.Bits.u1RF = 0;
12919	pCtx->rip += pDis->cbInstr;
12920	if (!fOldIF)
12921	{
12922	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
12923	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12924	}
12925	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
12926	if ( !fDbgStepping
12927	&& pCtx->eflags.Bits.u1TF)
12928	{
12929	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12930	AssertRCReturn(rc, rc);
12931	}
12932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12933	break;
12934	}
12935
12936	case OP_HLT:
12937	{
12938	rc = VINF_EM_HALT;
12939	pCtx->rip += pDis->cbInstr;
12940	pCtx->eflags.Bits.u1RF = 0;
12941	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
12942	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12943	break;
12944	}
12945
12946	case OP_POPF:
12947	{
12948	Log4Func(("POPF CS:EIP %04x:%04RX64\n", pCtx->cs.Sel, pCtx->rip));
12949	uint32_t cbParm;
12950	uint32_t uMask;
12951	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12952	if (pDis->fPrefix & DISPREFIX_OPSIZE)
12953	{
12954	cbParm = 4;
12955	uMask = 0xffffffff;
12956	}
12957	else
12958	{
12959	cbParm = 2;
12960	uMask = 0xffff;
12961	}
12962
12963	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
12964	RTGCPTR GCPtrStack = 0;
12965	X86EFLAGS Eflags;
12966	Eflags.u32 = 0;
12967	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12968	&GCPtrStack);
12969	if (RT_SUCCESS(rc))
12970	{
12971	Assert(sizeof(Eflags.u32) >= cbParm);
12972	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
12973	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
12974	}
12975	if (RT_FAILURE(rc))
12976	{
12977	rc = VERR_EM_INTERPRETER;
12978	break;
12979	}
12980	Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pCtx->rsp, uMask, pCtx->rip));
12981	pCtx->eflags.u32 = (pCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
12982	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
12983	pCtx->esp += cbParm;
12984	pCtx->esp &= uMask;
12985	pCtx->rip += pDis->cbInstr;
12986	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
12987	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
12988	POPF restores EFLAGS.TF. */
12989	if ( !fDbgStepping
12990	&& fGstStepping)
12991	{
12992	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12993	AssertRCReturn(rc, rc);
12994	}
12995	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
12996	break;
12997	}
12998
12999	case OP_PUSHF:
13000	{
13001	uint32_t cbParm;
13002	uint32_t uMask;
13003	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13004	{
13005	cbParm = 4;
13006	uMask = 0xffffffff;
13007	}
13008	else
13009	{
13010	cbParm = 2;
13011	uMask = 0xffff;
13012	}
13013
13014	/* Get the stack pointer & push the contents of eflags onto the stack. */
13015	RTGCPTR GCPtrStack = 0;
13016	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask,
13017	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13018	if (RT_FAILURE(rc))
13019	{
13020	rc = VERR_EM_INTERPRETER;
13021	break;
13022	}
13023	X86EFLAGS Eflags = pCtx->eflags;
13024	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13025	Eflags.Bits.u1RF = 0;
13026	Eflags.Bits.u1VM = 0;
13027
13028	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13029	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13030	{
13031	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13032	rc = VERR_EM_INTERPRETER;
13033	break;
13034	}
13035	Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13036	pCtx->esp -= cbParm;
13037	pCtx->esp &= uMask;
13038	pCtx->rip += pDis->cbInstr;
13039	pCtx->eflags.Bits.u1RF = 0;
13040	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
13041	if ( !fDbgStepping
13042	&& pCtx->eflags.Bits.u1TF)
13043	{
13044	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13045	AssertRCReturn(rc, rc);
13046	}
13047	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13048	break;
13049	}
13050
13051	case OP_IRET:
13052	{
13053	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13054	* instruction reference. */
13055	RTGCPTR GCPtrStack = 0;
13056	uint32_t uMask = 0xffff;
13057	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13058	uint16_t aIretFrame[3];
13059	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13060	{
13061	rc = VERR_EM_INTERPRETER;
13062	break;
13063	}
13064	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13065	&GCPtrStack);
13066	if (RT_SUCCESS(rc))
13067	{
13068	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13069	PGMACCESSORIGIN_HM));
13070	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13071	}
13072	if (RT_FAILURE(rc))
13073	{
13074	rc = VERR_EM_INTERPRETER;
13075	break;
13076	}
13077	pCtx->eip = 0;
13078	pCtx->ip = aIretFrame[0];
13079	pCtx->cs.Sel = aIretFrame[1];
13080	pCtx->cs.ValidSel = aIretFrame[1];
13081	pCtx->cs.u64Base = (uint64_t)pCtx->cs.Sel << 4;
13082	pCtx->eflags.u32 = (pCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13083	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13084	pCtx->sp += sizeof(aIretFrame);
13085	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
13086	\| HM_CHANGED_GUEST_CS);
13087	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13088	if ( !fDbgStepping
13089	&& fGstStepping)
13090	{
13091	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13092	AssertRCReturn(rc, rc);
13093	}
13094	Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pCtx->cs.Sel, pCtx->ip));
13095	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13096	break;
13097	}
13098
13099	case OP_INT:
13100	{
13101	uint16_t uVector = pDis->Param1.uValue & 0xff;
13102	hmR0VmxSetPendingIntN(pVCpu, uVector, pDis->cbInstr);
13103	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13104	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13105	break;
13106	}
13107
13108	case OP_INTO:
13109	{
13110	if (pCtx->eflags.Bits.u1OF)
13111	{
13112	hmR0VmxSetPendingXcptOF(pVCpu, pDis->cbInstr);
13113	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13114	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13115	}
13116	else
13117	{
13118	pCtx->eflags.Bits.u1RF = 0;
13119	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13120	}
13121	break;
13122	}
13123
13124	default:
13125	{
13126	pCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13127	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0 /* pvFault */,
13128	EMCODETYPE_SUPERVISOR);
13129	rc = VBOXSTRICTRC_VAL(rc2);
13130	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13131	/** @todo We have to set pending-debug exceptions here when the guest is
13132	* single-stepping depending on the instruction that was interpreted. */
13133	Log4Func(("#GP rc=%Rrc\n", rc));
13134	break;
13135	}
13136	}
13137	}
13138	else
13139	rc = VERR_EM_INTERPRETER;
13140
13141	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13142	("#GP Unexpected rc=%Rrc\n", rc));
13143	return rc;
13144	}
13145
13146
13147	/**
13148	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13149	* the exception reported in the VMX transient structure back into the VM.
13150	*
13151	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13152	* up-to-date.
13153	*/
13154	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13155	{
13156	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13157	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13158	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13159	("uVector=%#x u32XcptBitmap=%#X32\n",
13160	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13161	#endif
13162
13163	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13164	hmR0VmxCheckExitDueToEventDelivery(). */
13165	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13166	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13167	AssertRCReturn(rc, rc);
13168	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13169
13170	#ifdef DEBUG_ramshankar
13171	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13172	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13173	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13174	#endif
13175
13176	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13177	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13178	return VINF_SUCCESS;
13179	}
13180
13181
13182	/**
13183	* VM-exit exception handler for \#PF (Page-fault exception).
13184	*/
13185	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13186	{
13187	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13188	PVM pVM = pVCpu->CTX_SUFF(pVM);
13189	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13190	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13191	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13192	AssertRCReturn(rc, rc);
13193
13194	if (!pVM->hm.s.fNestedPaging)
13195	{ /* likely */ }
13196	else
13197	{
13198	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13199	Assert(pVCpu->hm.s.fUsingDebugLoop);
13200	#endif
13201	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13202	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13203	{
13204	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13205	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13206	}
13207	else
13208	{
13209	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13210	hmR0VmxSetPendingXcptDF(pVCpu);
13211	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13212	}
13213	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13214	return rc;
13215	}
13216
13217	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13218	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13219	if (pVmxTransient->fVectoringPF)
13220	{
13221	Assert(pVCpu->hm.s.Event.fPending);
13222	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13223	}
13224
13225	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13226	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13227	AssertRCReturn(rc, rc);
13228
13229	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13230	pCtx->cs.Sel, pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13231
13232	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13233	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx),
13234	(RTGCPTR)pVmxTransient->uExitQualification);
13235
13236	Log4Func(("#PF: rc=%Rrc\n", rc));
13237	if (rc == VINF_SUCCESS)
13238	{
13239	/*
13240	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13241	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13242	*/
13243	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13244	TRPMResetTrap(pVCpu);
13245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13246	return rc;
13247	}
13248
13249	if (rc == VINF_EM_RAW_GUEST_TRAP)
13250	{
13251	if (!pVmxTransient->fVectoringDoublePF)
13252	{
13253	/* It's a guest page fault and needs to be reflected to the guest. */
13254	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13255	TRPMResetTrap(pVCpu);
13256	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13257	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13258	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13259	}
13260	else
13261	{
13262	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13263	TRPMResetTrap(pVCpu);
13264	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13265	hmR0VmxSetPendingXcptDF(pVCpu);
13266	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13267	}
13268
13269	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13270	return VINF_SUCCESS;
13271	}
13272
13273	TRPMResetTrap(pVCpu);
13274	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13275	return rc;
13276	}
13277
13278	/** @} */
13279

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72992

Download in other formats: