HMVMXR0.cpp@ 72872

Last change on this file since 72872 was 72872, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: bugref:9193 Fix a bug with our exception bitmap cache going out of sync with the VMCS value. Added additional consistency checks for VMCS cache values (exception bitmap, TSC offset) - now reports specific errors via u32HmError on VERR_VMX_INVALID_GUEST_STATE gurus.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 577.4 KB

Line
1	/* $Id: HMVMXR0.cpp 72872 2018-07-04 13:14:32Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/gim.h>
35	#include <VBox/vmm/apic.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#include "HMInternal.h"
40	#include <VBox/vmm/vm.h>
41	#include "HMVMXR0.h"
42	#include "dtrace/VBoxVMM.h"
43
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50	# define HMVMX_ALWAYS_TRAP_PF
51	# define HMVMX_ALWAYS_FLUSH_TLB
52	# define HMVMX_ALWAYS_SWAP_EFER
53	#endif
54
55
56	/*********************************************************************************************************************************
57	* Defined Constants And Macros *
58	*********************************************************************************************************************************/
59	/** Use the function table. */
60	#define HMVMX_USE_FUNCTION_TABLE
61
62	/** Determine which tagged-TLB flush handler to use. */
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68	/** @name HMVMX_READ_XXX
69	* Flags to skip redundant reads of some common VMCS fields that are not part of
70	* the guest-CPU or VCPU state but are needed while handling VM-exits.
71	*/
72	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79	/** @} */
80
81	/**
82	* States of the VMCS.
83	*
84	* This does not reflect all possible VMCS states but currently only those
85	* needed for maintaining the VMCS consistently even when thread-context hooks
86	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
87	*/
88	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92	/**
93	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94	* guest using hardware-assisted VMX.
95	*
96	* This excludes state like GPRs (other than RSP) which are always are
97	* swapped and restored across the world-switch and also registers like EFER,
98	* MSR which cannot be modified by the guest without causing a VM-exit.
99	*/
100	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101	\| CPUMCTX_EXTRN_RFLAGS \
102	\| CPUMCTX_EXTRN_RSP \
103	\| CPUMCTX_EXTRN_SREG_MASK \
104	\| CPUMCTX_EXTRN_TABLE_MASK \
105	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
106	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
107	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
108	\| CPUMCTX_EXTRN_TSC_AUX \
109	\| CPUMCTX_EXTRN_OTHER_MSRS \
110	\| CPUMCTX_EXTRN_CR0 \
111	\| CPUMCTX_EXTRN_CR3 \
112	\| CPUMCTX_EXTRN_CR4 \
113	\| CPUMCTX_EXTRN_DR7 \
114	\| CPUMCTX_EXTRN_HM_VMX_MASK)
115
116	/**
117	* Exception bitmap mask for real-mode guests (real-on-v86).
118	*
119	* We need to intercept all exceptions manually except:
120	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121	* due to bugs in Intel CPUs.
122	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
123	* support.
124	*/
125	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
126	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
127	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
128	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
129	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
131	\| RT_BIT(X86_XCPT_XF))
132
133	/** Maximum VM-instruction error number. */
134	#define HMVMX_INSTR_ERROR_MAX 28
135
136	/** Profiling macro. */
137	#ifdef HM_PROFILE_EXIT_DISPATCH
138	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140	#else
141	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143	#endif
144
145	/** Assert that preemption is disabled or covered by thread-context hooks. */
146	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
147	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149	/** Assert that we haven't migrated CPUs when thread-context hooks are not
150	* used. */
151	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
152	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
153	("Illegal migration! Entered on CPU %u Current %u\n", \
154	pVCpu->hm.s.idEnteredCpu, RTMpCpuId()))
155
156	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157	* context. */
158	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162	/** Helper macro for VM-exit handlers called unexpectedly. */
163	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164	do { \
165	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166	return VERR_VMX_UNEXPECTED_EXIT; \
167	} while (0)
168
169	/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171	# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172	hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174	#else
175	# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176	hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178	#endif
179
180	/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181	# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182	hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186	/*********************************************************************************************************************************
187	* Structures and Typedefs *
188	*********************************************************************************************************************************/
189	/**
190	* VMX transient state.
191	*
192	* A state structure for holding miscellaneous information across
193	* VMX non-root operation and restored after the transition.
194	*/
195	typedef struct VMXTRANSIENT
196	{
197	/** The host's rflags/eflags. */
198	RTCCUINTREG fEFlags;
199	#if HC_ARCH_BITS == 32
200	uint32_t u32Alignment0;
201	#endif
202	/** The guest's TPR value used for TPR shadowing. */
203	uint8_t u8GuestTpr;
204	/** Alignment. */
205	uint8_t abAlignment0[7];
206
207	/** The basic VM-exit reason. */
208	uint16_t uExitReason;
209	/** Alignment. */
210	uint16_t u16Alignment0;
211	/** The VM-exit interruption error code. */
212	uint32_t uExitIntErrorCode;
213	/** The VM-exit exit code qualification. */
214	uint64_t uExitQualification;
215
216	/** The VM-exit interruption-information field. */
217	uint32_t uExitIntInfo;
218	/** The VM-exit instruction-length field. */
219	uint32_t cbInstr;
220	/** The VM-exit instruction-information field. */
221	union
222	{
223	/** Plain unsigned int representation. */
224	uint32_t u;
225	/** INS and OUTS information. */
226	struct
227	{
228	uint32_t u7Reserved0 : 7;
229	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230	uint32_t u3AddrSize : 3;
231	uint32_t u5Reserved1 : 5;
232	/** The segment register (X86_SREG_XXX). */
233	uint32_t iSegReg : 3;
234	uint32_t uReserved2 : 14;
235	} StrIo;
236	/** INVEPT, INVVPID, INVPCID information. */
237	struct
238	{
239	/** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240	uint32_t u2Scaling : 2;
241	uint32_t u5Reserved0 : 5;
242	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243	uint32_t u3AddrSize : 3;
244	uint32_t u1Reserved0 : 1;
245	uint32_t u4Reserved0 : 4;
246	/** The segment register (X86_SREG_XXX). */
247	uint32_t iSegReg : 3;
248	/** The index register (X86_GREG_XXX). */
249	uint32_t iIdxReg : 4;
250	/** Set if index register is invalid. */
251	uint32_t fIdxRegValid : 1;
252	/** The base register (X86_GREG_XXX). */
253	uint32_t iBaseReg : 4;
254	/** Set if base register is invalid. */
255	uint32_t fBaseRegValid : 1;
256	/** Register 2 (X86_GREG_XXX). */
257	uint32_t iReg2 : 4;
258	} Inv;
259	} ExitInstrInfo;
260	/** Whether the VM-entry failed or not. */
261	bool fVMEntryFailed;
262	/** Alignment. */
263	uint8_t abAlignment1[3];
264
265	/** The VM-entry interruption-information field. */
266	uint32_t uEntryIntInfo;
267	/** The VM-entry exception error code field. */
268	uint32_t uEntryXcptErrorCode;
269	/** The VM-entry instruction length field. */
270	uint32_t cbEntryInstr;
271
272	/** IDT-vectoring information field. */
273	uint32_t uIdtVectoringInfo;
274	/** IDT-vectoring error code. */
275	uint32_t uIdtVectoringErrorCode;
276
277	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278	uint32_t fVmcsFieldsRead;
279
280	/** Whether the guest debug state was active at the time of VM-exit. */
281	bool fWasGuestDebugStateActive;
282	/** Whether the hyper debug state was active at the time of VM-exit. */
283	bool fWasHyperDebugStateActive;
284	/** Whether TSC-offsetting should be setup before VM-entry. */
285	bool fUpdateTscOffsettingAndPreemptTimer;
286	/** Whether the VM-exit was caused by a page-fault during delivery of a
287	* contributory exception or a page-fault. */
288	bool fVectoringDoublePF;
289	/** Whether the VM-exit was caused by a page-fault during delivery of an
290	* external interrupt or NMI. */
291	bool fVectoringPF;
292	} VMXTRANSIENT;
293	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298	/** Pointer to VMX transient state. */
299	typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302	/**
303	* MSR-bitmap read permissions.
304	*/
305	typedef enum VMXMSREXITREAD
306	{
307	/** Reading this MSR causes a VM-exit. */
308	VMXMSREXIT_INTERCEPT_READ = 0xb,
309	/** Reading this MSR does not cause a VM-exit. */
310	VMXMSREXIT_PASSTHRU_READ
311	} VMXMSREXITREAD;
312	/** Pointer to MSR-bitmap read permissions. */
313	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315	/**
316	* MSR-bitmap write permissions.
317	*/
318	typedef enum VMXMSREXITWRITE
319	{
320	/** Writing to this MSR causes a VM-exit. */
321	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322	/** Writing to this MSR does not cause a VM-exit. */
323	VMXMSREXIT_PASSTHRU_WRITE
324	} VMXMSREXITWRITE;
325	/** Pointer to MSR-bitmap write permissions. */
326	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329	/**
330	* VMX VM-exit handler.
331	*
332	* @returns Strict VBox status code (i.e. informational status codes too).
333	* @param pVCpu The cross context virtual CPU structure.
334	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
335	* out-of-sync. Make sure to update the required
336	* fields before using them.
337	* @param pVmxTransient Pointer to the VMX-transient structure.
338	*/
339	#ifndef HMVMX_USE_FUNCTION_TABLE
340	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
341	#else
342	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
343	/** Pointer to VM-exit handler. */
344	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
345	#endif
346
347	/**
348	* VMX VM-exit handler, non-strict status code.
349	*
350	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
351	*
352	* @returns VBox status code, no informational status code returned.
353	* @param pVCpu The cross context virtual CPU structure.
354	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
355	* out-of-sync. Make sure to update the required
356	* fields before using them.
357	* @param pVmxTransient Pointer to the VMX-transient structure.
358	*
359	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
360	* use of that status code will be replaced with VINF_EM_SOMETHING
361	* later when switching over to IEM.
362	*/
363	#ifndef HMVMX_USE_FUNCTION_TABLE
364	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
365	#else
366	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
367	#endif
368
369
370	/*********************************************************************************************************************************
371	* Internal Functions *
372	*********************************************************************************************************************************/
373	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
374	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
375	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
376	static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
377	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
378	RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
379	#if HC_ARCH_BITS == 32
380	static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
381	#endif
382	#ifndef HMVMX_USE_FUNCTION_TABLE
383	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
384	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
385	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
386	#else
387	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
388	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
389	#endif
390
391
392	/** @name VM-exit handlers.
393	* @{
394	*/
395	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
396	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
397	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
400	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
401	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
404	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
405	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
406	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
407	static FNVMXEXITHANDLER hmR0VmxExitHlt;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
409	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
410	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
411	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
412	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
415	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
416	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
417	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
418	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
419	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
422	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
423	static FNVMXEXITHANDLER hmR0VmxExitMwait;
424	static FNVMXEXITHANDLER hmR0VmxExitMtf;
425	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
426	static FNVMXEXITHANDLER hmR0VmxExitPause;
427	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
428	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
429	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
430	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
432	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
433	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
434	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
435	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
436	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
437	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
438	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
439	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
440	/** @} */
441
442	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
449	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
450
451
452	/*********************************************************************************************************************************
453	* Global Variables *
454	*********************************************************************************************************************************/
455	#ifdef HMVMX_USE_FUNCTION_TABLE
456
457	/**
458	* VMX_EXIT dispatch table.
459	*/
460	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
461	{
462	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
463	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
464	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
465	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
466	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
467	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
468	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
469	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
470	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
471	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
472	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
473	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
474	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
475	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
476	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
477	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
478	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
479	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
480	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
481	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
482	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
483	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
484	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
485	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
486	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
487	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
488	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
489	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
490	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
491	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
492	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
493	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
494	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
495	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
496	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
497	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
498	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
499	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
500	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
501	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
502	/* 40 UNDEFINED */ hmR0VmxExitPause,
503	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
504	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
505	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
506	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
507	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
508	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
509	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
510	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
511	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
512	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
513	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
514	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
515	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
516	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
517	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
518	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
519	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
520	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
521	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
522	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
523	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
524	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
525	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
526	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
527	};
528	#endif /* HMVMX_USE_FUNCTION_TABLE */
529
530	#ifdef VBOX_STRICT
531	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
532	{
533	/* 0 */ "(Not Used)",
534	/* 1 */ "VMCALL executed in VMX root operation.",
535	/* 2 */ "VMCLEAR with invalid physical address.",
536	/* 3 */ "VMCLEAR with VMXON pointer.",
537	/* 4 */ "VMLAUNCH with non-clear VMCS.",
538	/* 5 */ "VMRESUME with non-launched VMCS.",
539	/* 6 */ "VMRESUME after VMXOFF",
540	/* 7 */ "VM-entry with invalid control fields.",
541	/* 8 */ "VM-entry with invalid host state fields.",
542	/* 9 */ "VMPTRLD with invalid physical address.",
543	/* 10 */ "VMPTRLD with VMXON pointer.",
544	/* 11 */ "VMPTRLD with incorrect revision identifier.",
545	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
546	/* 13 */ "VMWRITE to read-only VMCS component.",
547	/* 14 */ "(Not Used)",
548	/* 15 */ "VMXON executed in VMX root operation.",
549	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
550	/* 17 */ "VM-entry with non-launched executing VMCS.",
551	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
552	/* 19 */ "VMCALL with non-clear VMCS.",
553	/* 20 */ "VMCALL with invalid VM-exit control fields.",
554	/* 21 */ "(Not Used)",
555	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
556	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
557	/* 24 */ "VMCALL with invalid SMM-monitor features.",
558	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
559	/* 26 */ "VM-entry with events blocked by MOV SS.",
560	/* 27 */ "(Not Used)",
561	/* 28 */ "Invalid operand to INVEPT/INVVPID."
562	};
563	#endif /* VBOX_STRICT */
564
565
566
567	/**
568	* Updates the VM's last error record.
569	*
570	* If there was a VMX instruction error, reads the error data from the VMCS and
571	* updates VCPU's last error record as well.
572	*
573	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
574	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
575	* VERR_VMX_INVALID_VMCS_FIELD.
576	* @param rc The error code.
577	*/
578	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
579	{
580	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
581	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
582	{
583	AssertPtrReturnVoid(pVCpu);
584	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
585	}
586	pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
587	}
588
589
590	/**
591	* Reads the VM-entry interruption-information field from the VMCS into the VMX
592	* transient structure.
593	*
594	* @returns VBox status code.
595	* @param pVmxTransient Pointer to the VMX transient structure.
596	*
597	* @remarks No-long-jump zone!!!
598	*/
599	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
600	{
601	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
602	AssertRCReturn(rc, rc);
603	return VINF_SUCCESS;
604	}
605
606	#ifdef VBOX_STRICT
607	/**
608	* Reads the VM-entry exception error code field from the VMCS into
609	* the VMX transient structure.
610	*
611	* @returns VBox status code.
612	* @param pVmxTransient Pointer to the VMX transient structure.
613	*
614	* @remarks No-long-jump zone!!!
615	*/
616	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
617	{
618	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
619	AssertRCReturn(rc, rc);
620	return VINF_SUCCESS;
621	}
622
623
624	/**
625	* Reads the VM-entry exception error code field from the VMCS into
626	* the VMX transient structure.
627	*
628	* @returns VBox status code.
629	* @param pVmxTransient Pointer to the VMX transient structure.
630	*
631	* @remarks No-long-jump zone!!!
632	*/
633	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
634	{
635	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
636	AssertRCReturn(rc, rc);
637	return VINF_SUCCESS;
638	}
639	#endif /* VBOX_STRICT */
640
641
642	/**
643	* Reads the VM-exit interruption-information field from the VMCS into the VMX
644	* transient structure.
645	*
646	* @returns VBox status code.
647	* @param pVmxTransient Pointer to the VMX transient structure.
648	*/
649	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
650	{
651	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
652	{
653	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
654	AssertRCReturn(rc,rc);
655	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
656	}
657	return VINF_SUCCESS;
658	}
659
660
661	/**
662	* Reads the VM-exit interruption error code from the VMCS into the VMX
663	* transient structure.
664	*
665	* @returns VBox status code.
666	* @param pVmxTransient Pointer to the VMX transient structure.
667	*/
668	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
669	{
670	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
671	{
672	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
673	AssertRCReturn(rc, rc);
674	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
675	}
676	return VINF_SUCCESS;
677	}
678
679
680	/**
681	* Reads the VM-exit instruction length field from the VMCS into the VMX
682	* transient structure.
683	*
684	* @returns VBox status code.
685	* @param pVmxTransient Pointer to the VMX transient structure.
686	*/
687	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
688	{
689	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
690	{
691	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
692	AssertRCReturn(rc, rc);
693	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
694	}
695	return VINF_SUCCESS;
696	}
697
698
699	/**
700	* Reads the VM-exit instruction-information field from the VMCS into
701	* the VMX transient structure.
702	*
703	* @returns VBox status code.
704	* @param pVmxTransient Pointer to the VMX transient structure.
705	*/
706	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
707	{
708	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
709	{
710	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
711	AssertRCReturn(rc, rc);
712	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
713	}
714	return VINF_SUCCESS;
715	}
716
717
718	/**
719	* Reads the exit code qualification from the VMCS into the VMX transient
720	* structure.
721	*
722	* @returns VBox status code.
723	* @param pVCpu The cross context virtual CPU structure of the
724	* calling EMT. (Required for the VMCS cache case.)
725	* @param pVmxTransient Pointer to the VMX transient structure.
726	*/
727	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
728	{
729	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
730	{
731	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
732	AssertRCReturn(rc, rc);
733	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
734	}
735	return VINF_SUCCESS;
736	}
737
738
739	/**
740	* Reads the IDT-vectoring information field from the VMCS into the VMX
741	* transient structure.
742	*
743	* @returns VBox status code.
744	* @param pVmxTransient Pointer to the VMX transient structure.
745	*
746	* @remarks No-long-jump zone!!!
747	*/
748	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
749	{
750	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
751	{
752	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
753	AssertRCReturn(rc, rc);
754	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
755	}
756	return VINF_SUCCESS;
757	}
758
759
760	/**
761	* Reads the IDT-vectoring error code from the VMCS into the VMX
762	* transient structure.
763	*
764	* @returns VBox status code.
765	* @param pVmxTransient Pointer to the VMX transient structure.
766	*/
767	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
768	{
769	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
770	{
771	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
772	AssertRCReturn(rc, rc);
773	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
774	}
775	return VINF_SUCCESS;
776	}
777
778
779	/**
780	* Enters VMX root mode operation on the current CPU.
781	*
782	* @returns VBox status code.
783	* @param pVM The cross context VM structure. Can be
784	* NULL, after a resume.
785	* @param HCPhysCpuPage Physical address of the VMXON region.
786	* @param pvCpuPage Pointer to the VMXON region.
787	*/
788	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
789	{
790	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
791	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
792	Assert(pvCpuPage);
793	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
794
795	if (pVM)
796	{
797	/* Write the VMCS revision dword to the VMXON region. */
798	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
799	}
800
801	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
802	RTCCUINTREG fEFlags = ASMIntDisableFlags();
803
804	/* Enable the VMX bit in CR4 if necessary. */
805	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
806
807	/* Enter VMX root mode. */
808	int rc = VMXEnable(HCPhysCpuPage);
809	if (RT_FAILURE(rc))
810	{
811	if (!(uOldCr4 & X86_CR4_VMXE))
812	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
813
814	if (pVM)
815	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
816	}
817
818	/* Restore interrupts. */
819	ASMSetFlags(fEFlags);
820	return rc;
821	}
822
823
824	/**
825	* Exits VMX root mode operation on the current CPU.
826	*
827	* @returns VBox status code.
828	*/
829	static int hmR0VmxLeaveRootMode(void)
830	{
831	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
832
833	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
834	RTCCUINTREG fEFlags = ASMIntDisableFlags();
835
836	/* If we're for some reason not in VMX root mode, then don't leave it. */
837	RTCCUINTREG uHostCR4 = ASMGetCR4();
838
839	int rc;
840	if (uHostCR4 & X86_CR4_VMXE)
841	{
842	/* Exit VMX root mode and clear the VMX bit in CR4. */
843	VMXDisable();
844	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
845	rc = VINF_SUCCESS;
846	}
847	else
848	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
849
850	/* Restore interrupts. */
851	ASMSetFlags(fEFlags);
852	return rc;
853	}
854
855
856	/**
857	* Allocates and maps one physically contiguous page. The allocated page is
858	* zero'd out. (Used by various VT-x structures).
859	*
860	* @returns IPRT status code.
861	* @param pMemObj Pointer to the ring-0 memory object.
862	* @param ppVirt Where to store the virtual address of the
863	* allocation.
864	* @param pHCPhys Where to store the physical address of the
865	* allocation.
866	*/
867	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
868	{
869	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
870	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
871	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
872
873	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
874	if (RT_FAILURE(rc))
875	return rc;
876	ppVirt = RTR0MemObjAddress(pMemObj);
877	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
878	ASMMemZero32(*ppVirt, PAGE_SIZE);
879	return VINF_SUCCESS;
880	}
881
882
883	/**
884	* Frees and unmaps an allocated physical page.
885	*
886	* @param pMemObj Pointer to the ring-0 memory object.
887	* @param ppVirt Where to re-initialize the virtual address of
888	* allocation as 0.
889	* @param pHCPhys Where to re-initialize the physical address of the
890	* allocation as 0.
891	*/
892	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
893	{
894	AssertPtr(pMemObj);
895	AssertPtr(ppVirt);
896	AssertPtr(pHCPhys);
897	if (*pMemObj != NIL_RTR0MEMOBJ)
898	{
899	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
900	AssertRC(rc);
901	*pMemObj = NIL_RTR0MEMOBJ;
902	*ppVirt = 0;
903	*pHCPhys = 0;
904	}
905	}
906
907
908	/**
909	* Worker function to free VT-x related structures.
910	*
911	* @returns IPRT status code.
912	* @param pVM The cross context VM structure.
913	*/
914	static void hmR0VmxStructsFree(PVM pVM)
915	{
916	for (VMCPUID i = 0; i < pVM->cCpus; i++)
917	{
918	PVMCPU pVCpu = &pVM->aCpus[i];
919	AssertPtr(pVCpu);
920
921	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
922	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
923
924	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
925	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
926
927	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
928	}
929
930	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
931	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
932	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
933	#endif
934	}
935
936
937	/**
938	* Worker function to allocate VT-x related VM structures.
939	*
940	* @returns IPRT status code.
941	* @param pVM The cross context VM structure.
942	*/
943	static int hmR0VmxStructsAlloc(PVM pVM)
944	{
945	/*
946	* Initialize members up-front so we can cleanup properly on allocation failure.
947	*/
948	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
949	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951	pVM->hm.s.vmx.HCPhys##a_Name = 0;
952
953	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
954	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
955	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
956	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
957
958	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
959	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
960	#endif
961	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
962
963	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
964	for (VMCPUID i = 0; i < pVM->cCpus; i++)
965	{
966	PVMCPU pVCpu = &pVM->aCpus[i];
967	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
968	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
969	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
970	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
971	}
972	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
973	#undef VMXLOCAL_INIT_VM_MEMOBJ
974
975	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
976	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
977	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
978	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
979
980	/*
981	* Allocate all the VT-x structures.
982	*/
983	int rc = VINF_SUCCESS;
984	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
985	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
989	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
990	#endif
991
992	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
993	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
994	{
995	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
996	&pVM->hm.s.vmx.HCPhysApicAccess);
997	if (RT_FAILURE(rc))
998	goto cleanup;
999	}
1000
1001	/*
1002	* Initialize per-VCPU VT-x structures.
1003	*/
1004	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1005	{
1006	PVMCPU pVCpu = &pVM->aCpus[i];
1007	AssertPtr(pVCpu);
1008
1009	/* Allocate the VM control structure (VMCS). */
1010	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1011	if (RT_FAILURE(rc))
1012	goto cleanup;
1013
1014	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1015	if ( PDMHasApic(pVM)
1016	&& (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1017	{
1018	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1019	NULL /* pR3Ptr /, NULL / pRCPtr */);
1020	if (RT_FAILURE(rc))
1021	goto cleanup;
1022	}
1023
1024	/*
1025	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1026	* transparent accesses of specific MSRs.
1027	*
1028	* If the condition for enabling MSR bitmaps changes here, don't forget to
1029	* update HMAreMsrBitmapsAvailable().
1030	*/
1031	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1032	{
1033	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1034	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1035	if (RT_FAILURE(rc))
1036	goto cleanup;
1037	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1038	}
1039
1040	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1041	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1042	if (RT_FAILURE(rc))
1043	goto cleanup;
1044
1045	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1046	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1047	if (RT_FAILURE(rc))
1048	goto cleanup;
1049	}
1050
1051	return VINF_SUCCESS;
1052
1053	cleanup:
1054	hmR0VmxStructsFree(pVM);
1055	return rc;
1056	}
1057
1058
1059	/**
1060	* Does global VT-x initialization (called during module initialization).
1061	*
1062	* @returns VBox status code.
1063	*/
1064	VMMR0DECL(int) VMXR0GlobalInit(void)
1065	{
1066	#ifdef HMVMX_USE_FUNCTION_TABLE
1067	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1068	# ifdef VBOX_STRICT
1069	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1070	Assert(g_apfnVMExitHandlers[i]);
1071	# endif
1072	#endif
1073	return VINF_SUCCESS;
1074	}
1075
1076
1077	/**
1078	* Does global VT-x termination (called during module termination).
1079	*/
1080	VMMR0DECL(void) VMXR0GlobalTerm()
1081	{
1082	/* Nothing to do currently. */
1083	}
1084
1085
1086	/**
1087	* Sets up and activates VT-x on the current CPU.
1088	*
1089	* @returns VBox status code.
1090	* @param pHostCpu Pointer to the global CPU info struct.
1091	* @param pVM The cross context VM structure. Can be
1092	* NULL after a host resume operation.
1093	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1094	* fEnabledByHost is @c true).
1095	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1096	* @a fEnabledByHost is @c true).
1097	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1098	* enable VT-x on the host.
1099	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1100	*/
1101	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1102	void *pvMsrs)
1103	{
1104	Assert(pHostCpu);
1105	Assert(pvMsrs);
1106	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1107
1108	/* Enable VT-x if it's not already enabled by the host. */
1109	if (!fEnabledByHost)
1110	{
1111	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1112	if (RT_FAILURE(rc))
1113	return rc;
1114	}
1115
1116	/*
1117	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1118	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1119	* invalidated when flushing by VPID.
1120	*/
1121	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1122	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1123	{
1124	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1125	pHostCpu->fFlushAsidBeforeUse = false;
1126	}
1127	else
1128	pHostCpu->fFlushAsidBeforeUse = true;
1129
1130	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1131	++pHostCpu->cTlbFlushes;
1132
1133	return VINF_SUCCESS;
1134	}
1135
1136
1137	/**
1138	* Deactivates VT-x on the current CPU.
1139	*
1140	* @returns VBox status code.
1141	* @param pHostCpu Pointer to the global CPU info struct.
1142	* @param pvCpuPage Pointer to the VMXON region.
1143	* @param HCPhysCpuPage Physical address of the VMXON region.
1144	*
1145	* @remarks This function should never be called when SUPR0EnableVTx() or
1146	* similar was used to enable VT-x on the host.
1147	*/
1148	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1149	{
1150	RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1151
1152	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1153	return hmR0VmxLeaveRootMode();
1154	}
1155
1156
1157	/**
1158	* Sets the permission bits for the specified MSR in the MSR bitmap.
1159	*
1160	* @param pVCpu The cross context virtual CPU structure.
1161	* @param uMsr The MSR value.
1162	* @param enmRead Whether reading this MSR causes a VM-exit.
1163	* @param enmWrite Whether writing this MSR causes a VM-exit.
1164	*/
1165	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1166	{
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/*
1171	* Layout:
1172	* 0x000 - 0x3ff - Low MSR read bits
1173	* 0x400 - 0x7ff - High MSR read bits
1174	* 0x800 - 0xbff - Low MSR write bits
1175	* 0xc00 - 0xfff - High MSR write bits
1176	*/
1177	if (uMsr <= 0x00001FFF)
1178	iBit = uMsr;
1179	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1180	{
1181	iBit = uMsr - UINT32_C(0xC0000000);
1182	pbMsrBitmap += 0x400;
1183	}
1184	else
1185	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1186
1187	Assert(iBit <= 0x1fff);
1188	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1189	ASMBitSet(pbMsrBitmap, iBit);
1190	else
1191	ASMBitClear(pbMsrBitmap, iBit);
1192
1193	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1194	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1195	else
1196	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1197	}
1198
1199
1200	#ifdef VBOX_STRICT
1201	/**
1202	* Gets the permission bits for the specified MSR in the MSR bitmap.
1203	*
1204	* @returns VBox status code.
1205	* @retval VINF_SUCCESS if the specified MSR is found.
1206	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1207	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1208	*
1209	* @param pVCpu The cross context virtual CPU structure.
1210	* @param uMsr The MSR.
1211	* @param penmRead Where to store the read permissions.
1212	* @param penmWrite Where to store the write permissions.
1213	*/
1214	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1215	{
1216	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1217	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1218	int32_t iBit;
1219	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1220
1221	/* See hmR0VmxSetMsrPermission() for the layout. */
1222	if (uMsr <= 0x00001FFF)
1223	iBit = uMsr;
1224	else if ( uMsr >= 0xC0000000
1225	&& uMsr <= 0xC0001FFF)
1226	{
1227	iBit = (uMsr - 0xC0000000);
1228	pbMsrBitmap += 0x400;
1229	}
1230	else
1231	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1232
1233	Assert(iBit <= 0x1fff);
1234	if (ASMBitTest(pbMsrBitmap, iBit))
1235	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1236	else
1237	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1238
1239	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1240	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1241	else
1242	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1243	return VINF_SUCCESS;
1244	}
1245	#endif /* VBOX_STRICT */
1246
1247
1248	/**
1249	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1250	* area.
1251	*
1252	* @returns VBox status code.
1253	* @param pVCpu The cross context virtual CPU structure.
1254	* @param cMsrs The number of MSRs.
1255	*/
1256	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1257	{
1258	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1259	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1260	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1261	{
1262	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1263	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1264	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1265	}
1266
1267	/* Update number of guest MSRs to load/store across the world-switch. */
1268	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1269	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1270
1271	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1272	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1273	AssertRCReturn(rc, rc);
1274
1275	/* Update the VCPU's copy of the MSR count. */
1276	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1277
1278	return VINF_SUCCESS;
1279	}
1280
1281
1282	/**
1283	* Adds a new (or updates the value of an existing) guest/host MSR
1284	* pair to be swapped during the world-switch as part of the
1285	* auto-load/store MSR area in the VMCS.
1286	*
1287	* @returns VBox status code.
1288	* @param pVCpu The cross context virtual CPU structure.
1289	* @param uMsr The MSR.
1290	* @param uGuestMsrValue Value of the guest MSR.
1291	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1292	* necessary.
1293	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1294	* its value was updated. Optional, can be NULL.
1295	*/
1296	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1297	bool *pfAddedAndUpdated)
1298	{
1299	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1300	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1301	uint32_t i;
1302	for (i = 0; i < cMsrs; i++)
1303	{
1304	if (pGuestMsr->u32Msr == uMsr)
1305	break;
1306	pGuestMsr++;
1307	}
1308
1309	bool fAdded = false;
1310	if (i == cMsrs)
1311	{
1312	++cMsrs;
1313	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1314	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1315
1316	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1317	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1318	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1319
1320	fAdded = true;
1321	}
1322
1323	/* Update the MSR values in the auto-load/store MSR area. */
1324	pGuestMsr->u32Msr = uMsr;
1325	pGuestMsr->u64Value = uGuestMsrValue;
1326
1327	/* Create/update the MSR slot in the host MSR area. */
1328	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1329	pHostMsr += i;
1330	pHostMsr->u32Msr = uMsr;
1331
1332	/*
1333	* Update the host MSR only when requested by the caller AND when we're
1334	* adding it to the auto-load/store area. Otherwise, it would have been
1335	* updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1336	*/
1337	bool fUpdatedMsrValue = false;
1338	if ( fAdded
1339	&& fUpdateHostMsr)
1340	{
1341	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1342	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1343	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1344	fUpdatedMsrValue = true;
1345	}
1346
1347	if (pfAddedAndUpdated)
1348	*pfAddedAndUpdated = fUpdatedMsrValue;
1349	return VINF_SUCCESS;
1350	}
1351
1352
1353	/**
1354	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1355	* auto-load/store MSR area in the VMCS.
1356	*
1357	* @returns VBox status code.
1358	* @param pVCpu The cross context virtual CPU structure.
1359	* @param uMsr The MSR.
1360	*/
1361	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1362	{
1363	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1364	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1365	for (uint32_t i = 0; i < cMsrs; i++)
1366	{
1367	/* Find the MSR. */
1368	if (pGuestMsr->u32Msr == uMsr)
1369	{
1370	/* If it's the last MSR, simply reduce the count. */
1371	if (i == cMsrs - 1)
1372	{
1373	--cMsrs;
1374	break;
1375	}
1376
1377	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1378	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1379	pLastGuestMsr += cMsrs - 1;
1380	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1381	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1382
1383	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1384	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1385	pLastHostMsr += cMsrs - 1;
1386	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1387	pHostMsr->u64Value = pLastHostMsr->u64Value;
1388	--cMsrs;
1389	break;
1390	}
1391	pGuestMsr++;
1392	}
1393
1394	/* Update the VMCS if the count changed (meaning the MSR was found). */
1395	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1396	{
1397	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1398	AssertRCReturn(rc, rc);
1399
1400	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1401	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1402	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1403
1404	Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1405	return VINF_SUCCESS;
1406	}
1407
1408	return VERR_NOT_FOUND;
1409	}
1410
1411
1412	/**
1413	* Checks if the specified guest MSR is part of the auto-load/store area in
1414	* the VMCS.
1415	*
1416	* @returns true if found, false otherwise.
1417	* @param pVCpu The cross context virtual CPU structure.
1418	* @param uMsr The MSR to find.
1419	*/
1420	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1421	{
1422	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1423	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1424
1425	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1426	{
1427	if (pGuestMsr->u32Msr == uMsr)
1428	return true;
1429	}
1430	return false;
1431	}
1432
1433
1434	/**
1435	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1436	*
1437	* @param pVCpu The cross context virtual CPU structure.
1438	*
1439	* @remarks No-long-jump zone!!!
1440	*/
1441	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1442	{
1443	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1444	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1445	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1446	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1447
1448	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1449	{
1450	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1451
1452	/*
1453	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1454	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1455	*/
1456	if (pHostMsr->u32Msr == MSR_K6_EFER)
1457	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1458	else
1459	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1460	}
1461
1462	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1463	}
1464
1465
1466	/**
1467	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1468	* perform lazy restoration of the host MSRs while leaving VT-x.
1469	*
1470	* @param pVCpu The cross context virtual CPU structure.
1471	*
1472	* @remarks No-long-jump zone!!!
1473	*/
1474	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1475	{
1476	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1477
1478	/*
1479	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1480	*/
1481	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1482	{
1483	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1484	#if HC_ARCH_BITS == 64
1485	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1486	{
1487	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1488	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1489	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1490	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1491	}
1492	#endif
1493	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1494	}
1495	}
1496
1497
1498	/**
1499	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1500	* lazily while leaving VT-x.
1501	*
1502	* @returns true if it does, false otherwise.
1503	* @param pVCpu The cross context virtual CPU structure.
1504	* @param uMsr The MSR to check.
1505	*/
1506	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1507	{
1508	NOREF(pVCpu);
1509	#if HC_ARCH_BITS == 64
1510	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1511	{
1512	switch (uMsr)
1513	{
1514	case MSR_K8_LSTAR:
1515	case MSR_K6_STAR:
1516	case MSR_K8_SF_MASK:
1517	case MSR_K8_KERNEL_GS_BASE:
1518	return true;
1519	}
1520	}
1521	#else
1522	RT_NOREF(pVCpu, uMsr);
1523	#endif
1524	return false;
1525	}
1526
1527
1528	/**
1529	* Loads a set of guests MSRs to allow read/passthru to the guest.
1530	*
1531	* The name of this function is slightly confusing. This function does NOT
1532	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1533	* common prefix for functions dealing with "lazy restoration" of the shared
1534	* MSRs.
1535	*
1536	* @param pVCpu The cross context virtual CPU structure.
1537	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1538	* out-of-sync. Make sure to update the required fields
1539	* before using them.
1540	*
1541	* @remarks No-long-jump zone!!!
1542	*/
1543	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1544	{
1545	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1546	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1547
1548	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1549	#if HC_ARCH_BITS == 64
1550	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1551	{
1552	/*
1553	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1554	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1555	* we can skip a few MSR writes.
1556	*
1557	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1558	* guest MSR values in the guest-CPU context might be different to what's currently
1559	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1560	* CPU, see @bugref{8728}.
1561	*/
1562	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1563	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1564	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1565	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1566	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1567	{
1568	#ifdef VBOX_STRICT
1569	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1570	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1571	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1572	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1573	#endif
1574	}
1575	else
1576	{
1577	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1578	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1579	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1580	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1581	}
1582	}
1583	#else
1584	RT_NOREF(pMixedCtx);
1585	#endif
1586	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1587	}
1588
1589
1590	/**
1591	* Performs lazy restoration of the set of host MSRs if they were previously
1592	* loaded with guest MSR values.
1593	*
1594	* @param pVCpu The cross context virtual CPU structure.
1595	*
1596	* @remarks No-long-jump zone!!!
1597	* @remarks The guest MSRs should have been saved back into the guest-CPU
1598	* context by hmR0VmxImportGuestState()!!!
1599	*/
1600	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1601	{
1602	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1603	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1604
1605	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1606	{
1607	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1608	#if HC_ARCH_BITS == 64
1609	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1610	{
1611	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1612	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1613	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1614	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1615	}
1616	#endif
1617	}
1618	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1619	}
1620
1621
1622	/**
1623	* Verifies that our cached values of the VMCS fields are all consistent with
1624	* what's actually present in the VMCS.
1625	*
1626	* @returns VBox status code.
1627	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1628	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1629	* VMCS content. HMCPU error-field is
1630	* updated, see VMX_VCI_XXX.
1631	* @param pVCpu The cross context virtual CPU structure.
1632	*/
1633	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1634	{
1635	uint32_t u32Val;
1636	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1637	AssertRCReturn(rc, rc);
1638	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1639	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1640	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1641	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1642
1643	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1644	AssertRCReturn(rc, rc);
1645	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1646	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1647	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1648	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1649
1650	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651	AssertRCReturn(rc, rc);
1652	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1653	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1654	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1655	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1656
1657	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1658	AssertRCReturn(rc, rc);
1659	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1660	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1661	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1662	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1663
1664	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1665	{
1666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1667	AssertRCReturn(rc, rc);
1668	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1669	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1670	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1671	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1672	}
1673
1674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1675	AssertRCReturn(rc, rc);
1676	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1677	("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1678	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1679	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1680
1681	uint64_t u64Val;
1682	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1683	AssertRCReturn(rc, rc);
1684	AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1685	("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1686	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1687	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1688
1689	return VINF_SUCCESS;
1690	}
1691
1692
1693	#ifdef VBOX_STRICT
1694	/**
1695	* Verifies that our cached host EFER value has not changed
1696	* since we cached it.
1697	*
1698	* @param pVCpu The cross context virtual CPU structure.
1699	*/
1700	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1701	{
1702	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1703
1704	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1705	{
1706	uint64_t u64Val;
1707	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1708	AssertRC(rc);
1709
1710	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1711	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1712	}
1713	}
1714
1715
1716	/**
1717	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1718	* VMCS are correct.
1719	*
1720	* @param pVCpu The cross context virtual CPU structure.
1721	*/
1722	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1723	{
1724	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1725
1726	/* Verify MSR counts in the VMCS are what we think it should be. */
1727	uint32_t cMsrs;
1728	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1729	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1730
1731	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1732	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1733
1734	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1735	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1736
1737	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1738	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1739	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1740	{
1741	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1742	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1743	pGuestMsr->u32Msr, cMsrs));
1744
1745	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1746	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1747	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1748
1749	/* Verify that the permissions are as expected in the MSR bitmap. */
1750	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1751	{
1752	VMXMSREXITREAD enmRead;
1753	VMXMSREXITWRITE enmWrite;
1754	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1755	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1756	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1757	{
1758	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1759	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1760	}
1761	else
1762	{
1763	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1764	pGuestMsr->u32Msr, cMsrs));
1765	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1766	pGuestMsr->u32Msr, cMsrs));
1767	}
1768	}
1769	}
1770	}
1771	#endif /* VBOX_STRICT */
1772
1773
1774	/**
1775	* Flushes the TLB using EPT.
1776	*
1777	* @returns VBox status code.
1778	* @param pVCpu The cross context virtual CPU structure of the calling
1779	* EMT. Can be NULL depending on @a enmTlbFlush.
1780	* @param enmTlbFlush Type of flush.
1781	*
1782	* @remarks Caller is responsible for making sure this function is called only
1783	* when NestedPaging is supported and providing @a enmTlbFlush that is
1784	* supported by the CPU.
1785	* @remarks Can be called with interrupts disabled.
1786	*/
1787	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1788	{
1789	uint64_t au64Descriptor[2];
1790	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1791	au64Descriptor[0] = 0;
1792	else
1793	{
1794	Assert(pVCpu);
1795	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1796	}
1797	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1798
1799	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1800	AssertMsg(rc == VINF_SUCCESS,
1801	("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1802
1803	if ( RT_SUCCESS(rc)
1804	&& pVCpu)
1805	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1806	}
1807
1808
1809	/**
1810	* Flushes the TLB using VPID.
1811	*
1812	* @returns VBox status code.
1813	* @param pVCpu The cross context virtual CPU structure of the calling
1814	* EMT. Can be NULL depending on @a enmTlbFlush.
1815	* @param enmTlbFlush Type of flush.
1816	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1817	* on @a enmTlbFlush).
1818	*
1819	* @remarks Can be called with interrupts disabled.
1820	*/
1821	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1822	{
1823	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1824
1825	uint64_t au64Descriptor[2];
1826	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1827	{
1828	au64Descriptor[0] = 0;
1829	au64Descriptor[1] = 0;
1830	}
1831	else
1832	{
1833	AssertPtr(pVCpu);
1834	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1835	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1836	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1837	au64Descriptor[1] = GCPtr;
1838	}
1839
1840	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1841	AssertMsg(rc == VINF_SUCCESS,
1842	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1843
1844	if ( RT_SUCCESS(rc)
1845	&& pVCpu)
1846	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1847	NOREF(rc);
1848	}
1849
1850
1851	/**
1852	* Invalidates a guest page by guest virtual address. Only relevant for
1853	* EPT/VPID, otherwise there is nothing really to invalidate.
1854	*
1855	* @returns VBox status code.
1856	* @param pVCpu The cross context virtual CPU structure.
1857	* @param GCVirt Guest virtual address of the page to invalidate.
1858	*/
1859	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1860	{
1861	AssertPtr(pVCpu);
1862	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1863
1864	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1865	if (!fFlushPending)
1866	{
1867	/*
1868	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1869	* the EPT case. See @bugref{6043} and @bugref{6177}.
1870	*
1871	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1872	* as this function maybe called in a loop with individual addresses.
1873	*/
1874	PVM pVM = pVCpu->CTX_SUFF(pVM);
1875	if (pVM->hm.s.vmx.fVpid)
1876	{
1877	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1878
1879	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1880	/*
1881	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1882	* where executing INVVPID outside 64-bit mode does not flush translations of
1883	* 64-bit linear addresses, see @bugref{6208#c72}.
1884	*/
1885	if (RT_HI_U32(GCVirt))
1886	fVpidFlush = false;
1887	#endif
1888
1889	if (fVpidFlush)
1890	{
1891	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1892	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1893	}
1894	else
1895	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1896	}
1897	else if (pVM->hm.s.fNestedPaging)
1898	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1899	}
1900
1901	return VINF_SUCCESS;
1902	}
1903
1904
1905	/**
1906	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1907	* case where neither EPT nor VPID is supported by the CPU.
1908	*
1909	* @param pVCpu The cross context virtual CPU structure.
1910	* @param pCpu Pointer to the global HM struct.
1911	*
1912	* @remarks Called with interrupts disabled.
1913	*/
1914	static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1915	{
1916	AssertPtr(pVCpu);
1917	AssertPtr(pCpu);
1918
1919	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921	Assert(pCpu->idCpu != NIL_RTCPUID);
1922	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924	pVCpu->hm.s.fForceTLBFlush = false;
1925	return;
1926	}
1927
1928
1929	/**
1930	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931	*
1932	* @param pVCpu The cross context virtual CPU structure.
1933	* @param pCpu Pointer to the global HM CPU struct.
1934	*
1935	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1936	* nomenclature. The reason is, to avoid confusion in compare statements
1937	* since the host-CPU copies are named "ASID".
1938	*
1939	* @remarks Called with interrupts disabled.
1940	*/
1941	static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942	{
1943	#ifdef VBOX_WITH_STATISTICS
1944	bool fTlbFlushed = false;
1945	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947	if (!fTlbFlushed) \
1948	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949	} while (0)
1950	#else
1951	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953	#endif
1954
1955	AssertPtr(pCpu);
1956	AssertPtr(pVCpu);
1957	Assert(pCpu->idCpu != NIL_RTCPUID);
1958
1959	PVM pVM = pVCpu->CTX_SUFF(pVM);
1960	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964	/*
1965	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
1966	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1967	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1968	* cannot reuse the current ASID anymore.
1969	*/
1970	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1971	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1972	{
1973	++pCpu->uCurrentAsid;
1974	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1975	{
1976	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1977	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1978	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1979	}
1980
1981	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1982	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1983	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1984
1985	/*
1986	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1987	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1988	*/
1989	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1990	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1991	HMVMX_SET_TAGGED_TLB_FLUSHED();
1992	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1993	}
1994
1995	/* Check for explicit TLB flushes. */
1996	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1997	{
1998	/*
1999	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2000	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2001	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2002	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2003	* mappings, see @bugref{6568}.
2004	*
2005	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2006	*/
2007	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
2008	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2009	HMVMX_SET_TAGGED_TLB_FLUSHED();
2010	}
2011
2012	pVCpu->hm.s.fForceTLBFlush = false;
2013	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2014
2015	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2016	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2017	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2018	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2019	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2020	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2021	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2022	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2023	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2024
2025	/* Update VMCS with the VPID. */
2026	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2027	AssertRC(rc);
2028
2029	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2030	}
2031
2032
2033	/**
2034	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2035	*
2036	* @returns VBox status code.
2037	* @param pVCpu The cross context virtual CPU structure.
2038	* @param pCpu Pointer to the global HM CPU struct.
2039	*
2040	* @remarks Called with interrupts disabled.
2041	*/
2042	static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2043	{
2044	AssertPtr(pVCpu);
2045	AssertPtr(pCpu);
2046	Assert(pCpu->idCpu != NIL_RTCPUID);
2047	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2048	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2049
2050	/*
2051	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2052	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2053	*/
2054	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2055	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2056	{
2057	pVCpu->hm.s.fForceTLBFlush = true;
2058	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2059	}
2060
2061	/* Check for explicit TLB flushes. */
2062	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2063	{
2064	pVCpu->hm.s.fForceTLBFlush = true;
2065	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2066	}
2067
2068	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2069	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2070
2071	if (pVCpu->hm.s.fForceTLBFlush)
2072	{
2073	hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2074	pVCpu->hm.s.fForceTLBFlush = false;
2075	}
2076	}
2077
2078
2079	/**
2080	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2081	*
2082	* @returns VBox status code.
2083	* @param pVCpu The cross context virtual CPU structure.
2084	* @param pCpu Pointer to the global HM CPU struct.
2085	*
2086	* @remarks Called with interrupts disabled.
2087	*/
2088	static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089	{
2090	AssertPtr(pVCpu);
2091	AssertPtr(pCpu);
2092	Assert(pCpu->idCpu != NIL_RTCPUID);
2093	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2094	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2095
2096	/*
2097	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2098	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2099	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2100	* cannot reuse the current ASID anymore.
2101	*/
2102	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104	{
2105	pVCpu->hm.s.fForceTLBFlush = true;
2106	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107	}
2108
2109	/* Check for explicit TLB flushes. */
2110	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111	{
2112	/*
2113	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2114	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2115	* fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2116	* include fExplicitFlush's too) - an obscure corner case.
2117	*/
2118	pVCpu->hm.s.fForceTLBFlush = true;
2119	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2120	}
2121
2122	PVM pVM = pVCpu->CTX_SUFF(pVM);
2123	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2124	if (pVCpu->hm.s.fForceTLBFlush)
2125	{
2126	++pCpu->uCurrentAsid;
2127	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2128	{
2129	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2130	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2131	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2132	}
2133
2134	pVCpu->hm.s.fForceTLBFlush = false;
2135	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2136	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2137	if (pCpu->fFlushAsidBeforeUse)
2138	{
2139	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2140	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2141	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2142	{
2143	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2144	pCpu->fFlushAsidBeforeUse = false;
2145	}
2146	else
2147	{
2148	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2149	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2150	}
2151	}
2152	}
2153
2154	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2155	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2156	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2157	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2158	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2159	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2160	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2161
2162	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2163	AssertRC(rc);
2164	}
2165
2166
2167	/**
2168	* Flushes the guest TLB entry based on CPU capabilities.
2169	*
2170	* @param pVCpu The cross context virtual CPU structure.
2171	* @param pCpu Pointer to the global HM CPU struct.
2172	*/
2173	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2174	{
2175	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2176	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2177	#endif
2178	PVM pVM = pVCpu->CTX_SUFF(pVM);
2179	switch (pVM->hm.s.vmx.enmTlbFlushType)
2180	{
2181	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2182	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2183	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2184	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2185	default:
2186	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2187	break;
2188	}
2189	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2190	}
2191
2192
2193	/**
2194	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2195	* TLB entries from the host TLB before VM-entry.
2196	*
2197	* @returns VBox status code.
2198	* @param pVM The cross context VM structure.
2199	*/
2200	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2201	{
2202	/*
2203	* Determine optimal flush type for Nested Paging.
2204	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2205	* guest execution (see hmR3InitFinalizeR0()).
2206	*/
2207	if (pVM->hm.s.fNestedPaging)
2208	{
2209	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2210	{
2211	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2212	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2213	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2214	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2215	else
2216	{
2217	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2218	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2219	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2220	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2221	}
2222
2223	/* Make sure the write-back cacheable memory type for EPT is supported. */
2224	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2225	{
2226	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2227	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2228	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2229	}
2230
2231	/* EPT requires a page-walk length of 4. */
2232	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2233	{
2234	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2235	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2236	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2237	}
2238	}
2239	else
2240	{
2241	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2242	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2243	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2244	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2245	}
2246	}
2247
2248	/*
2249	* Determine optimal flush type for VPID.
2250	*/
2251	if (pVM->hm.s.vmx.fVpid)
2252	{
2253	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2254	{
2255	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2256	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2257	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2258	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2259	else
2260	{
2261	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2262	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2263	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2264	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2265	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2266	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2267	pVM->hm.s.vmx.fVpid = false;
2268	}
2269	}
2270	else
2271	{
2272	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2273	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2274	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2275	pVM->hm.s.vmx.fVpid = false;
2276	}
2277	}
2278
2279	/*
2280	* Setup the handler for flushing tagged-TLBs.
2281	*/
2282	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2283	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2284	else if (pVM->hm.s.fNestedPaging)
2285	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2286	else if (pVM->hm.s.vmx.fVpid)
2287	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2288	else
2289	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2290	return VINF_SUCCESS;
2291	}
2292
2293
2294	/**
2295	* Sets up pin-based VM-execution controls in the VMCS.
2296	*
2297	* @returns VBox status code.
2298	* @param pVCpu The cross context virtual CPU structure.
2299	*
2300	* @remarks We don't really care about optimizing vmwrites here as it's done only
2301	* once per VM and hence we don't care about VMCS-field cache comparisons.
2302	*/
2303	static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2304	{
2305	PVM pVM = pVCpu->CTX_SUFF(pVM);
2306	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2307	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2308
2309	fVal \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2310	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2311
2312	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2313	fVal \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2314
2315	/* Enable the VMX preemption timer. */
2316	if (pVM->hm.s.vmx.fUsePreemptTimer)
2317	{
2318	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2319	fVal \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2320	}
2321
2322	#if 0
2323	/* Enable posted-interrupt processing. */
2324	if (pVM->hm.s.fPostedIntrs)
2325	{
2326	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328	fVal \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329	}
2330	#endif
2331
2332	if ((fVal & fZap) != fVal)
2333	{
2334	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2335	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2336	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2337	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2338	}
2339
2340	/* Commit it to the VMCS and update our cache. */
2341	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2342	AssertRCReturn(rc, rc);
2343	pVCpu->hm.s.vmx.u32PinCtls = fVal;
2344
2345	return VINF_SUCCESS;
2346	}
2347
2348
2349	/**
2350	* Sets up secondary processor-based VM-execution controls in the VMCS.
2351	*
2352	* @returns VBox status code.
2353	* @param pVCpu The cross context virtual CPU structure.
2354	*
2355	* @remarks We don't really care about optimizing vmwrites here as it's done only
2356	* once per VM and hence we don't care about VMCS-field cache comparisons.
2357	*/
2358	static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2359	{
2360	PVM pVM = pVCpu->CTX_SUFF(pVM);
2361	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2362	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2363
2364	/* WBINVD causes a VM-exit. */
2365	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2366	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2367
2368	/* Enable EPT (aka nested-paging). */
2369	if (pVM->hm.s.fNestedPaging)
2370	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2371
2372	/*
2373	* Enable the INVPCID instruction if supported by the hardware and we expose
2374	* it to the guest. Without this, guest executing INVPCID would cause a #UD.
2375	*/
2376	if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2377	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2378	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2379
2380	/* Enable VPID. */
2381	if (pVM->hm.s.vmx.fVpid)
2382	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2383
2384	/* Enable Unrestricted guest execution. */
2385	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2386	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2387
2388	#if 0
2389	if (pVM->hm.s.fVirtApicRegs)
2390	{
2391	/* Enable APIC-register virtualization. */
2392	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2393	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2394
2395	/* Enable virtual-interrupt delivery. */
2396	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2397	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2398	}
2399	#endif
2400
2401	/* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2402	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2403	* done dynamically. */
2404	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2405	{
2406	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2407	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2408	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2409	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2410	AssertRCReturn(rc, rc);
2411	}
2412
2413	/* Enable RDTSCP. */
2414	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2415	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2416
2417	/* Enable Pause-Loop exiting. */
2418	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2419	&& pVM->hm.s.vmx.cPleGapTicks
2420	&& pVM->hm.s.vmx.cPleWindowTicks)
2421	{
2422	fVal \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2423
2424	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2425	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2426	AssertRCReturn(rc, rc);
2427	}
2428
2429	if ((fVal & fZap) != fVal)
2430	{
2431	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2432	pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2433	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2434	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2435	}
2436
2437	/* Commit it to the VMCS and update our cache. */
2438	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2439	AssertRCReturn(rc, rc);
2440	pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2441
2442	return VINF_SUCCESS;
2443	}
2444
2445
2446	/**
2447	* Sets up processor-based VM-execution controls in the VMCS.
2448	*
2449	* @returns VBox status code.
2450	* @param pVCpu The cross context virtual CPU structure.
2451	*
2452	* @remarks We don't really care about optimizing vmwrites here as it's done only
2453	* once per VM and hence we don't care about VMCS-field cache comparisons.
2454	*/
2455	static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2456	{
2457	PVM pVM = pVCpu->CTX_SUFF(pVM);
2458	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2459	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2460
2461	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2462	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2463	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2464	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2465	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2466	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2467	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2468
2469	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2470	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2471	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2472	{
2473	LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2474	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2475	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2476	}
2477
2478	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2479	if (!pVM->hm.s.fNestedPaging)
2480	{
2481	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2482	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2483	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2484	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2485	}
2486
2487	/* Use TPR shadowing if supported by the CPU. */
2488	if ( PDMHasApic(pVM)
2489	&& pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2490	{
2491	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2492	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2493	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2494	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2495	AssertRCReturn(rc, rc);
2496
2497	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2498	/* CR8 writes cause a VM-exit based on TPR threshold. */
2499	Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2500	Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2501	}
2502	else
2503	{
2504	/*
2505	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2506	* Set this control only for 64-bit guests.
2507	*/
2508	if (pVM->hm.s.fAllow64BitGuests)
2509	{
2510	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2511	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2512	}
2513	}
2514
2515	/* Use MSR-bitmaps if supported by the CPU. */
2516	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2517	{
2518	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2519
2520	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2521	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2522	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2523	AssertRCReturn(rc, rc);
2524
2525	/*
2526	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2527	* automatically using dedicated fields in the VMCS.
2528	*/
2529	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2530	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2533	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2534	#if HC_ARCH_BITS == 64
2535	/*
2536	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2537	*/
2538	if (pVM->hm.s.fAllow64BitGuests)
2539	{
2540	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2541	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2542	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2543	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2544	}
2545	#endif
2546	/*
2547	* The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2548	* access (writes need to be executed without exiting, reds will #GP-fault anyway).
2549	*/
2550	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2551	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2552
2553	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2554	}
2555
2556	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2557	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2558	fVal \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2559
2560	if ((fVal & fZap) != fVal)
2561	{
2562	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2563	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2564	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2565	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2566	}
2567
2568	/* Commit it to the VMCS and update our cache. */
2569	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2570	AssertRCReturn(rc, rc);
2571	pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2572
2573	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2574	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2575	return hmR0VmxSetupProcCtls2(pVCpu);
2576
2577	/* Sanity check, should not really happen. */
2578	if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2579	{
2580	LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2581	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2582	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2583	}
2584
2585	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2586	return VINF_SUCCESS;
2587	}
2588
2589
2590	/**
2591	* Sets up miscellaneous (everything other than Pin & Processor-based
2592	* VM-execution) control fields in the VMCS.
2593	*
2594	* @returns VBox status code.
2595	* @param pVCpu The cross context virtual CPU structure.
2596	*/
2597	static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2598	{
2599	AssertPtr(pVCpu);
2600
2601	int rc = VERR_GENERAL_FAILURE;
2602
2603	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2604	#if 0
2605	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2606	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2607	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2608
2609	/*
2610	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2611	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2612	* We thus use the exception bitmap to control it rather than use both.
2613	*/
2614	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2615	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2616
2617	/* All IO & IOIO instructions cause VM-exits. */
2618	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2619	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2620
2621	/* Initialize the MSR-bitmap area. */
2622	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2623	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2624	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2625	AssertRCReturn(rc, rc);
2626	#endif
2627
2628	/* Setup MSR auto-load/store area. */
2629	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2630	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2631	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2632	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2633	AssertRCReturn(rc, rc);
2634
2635	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2636	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2637	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2638	AssertRCReturn(rc, rc);
2639
2640	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2641	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2642	AssertRCReturn(rc, rc);
2643
2644	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2645	#if 0
2646	/* Setup debug controls */
2647	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2648	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2649	AssertRCReturn(rc, rc);
2650	#endif
2651
2652	return rc;
2653	}
2654
2655
2656	/**
2657	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2658	*
2659	* We shall setup those exception intercepts that don't change during the
2660	* lifetime of the VM here. The rest are done dynamically while loading the
2661	* guest state.
2662	*
2663	* @returns VBox status code.
2664	* @param pVCpu The cross context virtual CPU structure.
2665	*/
2666	static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2667	{
2668	AssertPtr(pVCpu);
2669
2670	uint32_t uXcptBitmap;
2671
2672	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2673	uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2674
2675	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2676	and writes, and because recursive #DBs can cause the CPU hang, we must always
2677	intercept #DB. */
2678	uXcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2679
2680	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2681	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2682	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
2683
2684	/* Commit it to the VMCS. */
2685	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2686	AssertRCReturn(rc, rc);
2687
2688	/* Update our cache of the exception bitmap. */
2689	pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2690	return VINF_SUCCESS;
2691	}
2692
2693
2694	/**
2695	* Does per-VM VT-x initialization.
2696	*
2697	* @returns VBox status code.
2698	* @param pVM The cross context VM structure.
2699	*/
2700	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2701	{
2702	LogFlowFunc(("pVM=%p\n", pVM));
2703
2704	int rc = hmR0VmxStructsAlloc(pVM);
2705	if (RT_FAILURE(rc))
2706	{
2707	LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2708	return rc;
2709	}
2710
2711	return VINF_SUCCESS;
2712	}
2713
2714
2715	/**
2716	* Does per-VM VT-x termination.
2717	*
2718	* @returns VBox status code.
2719	* @param pVM The cross context VM structure.
2720	*/
2721	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2722	{
2723	LogFlowFunc(("pVM=%p\n", pVM));
2724
2725	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2726	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2727	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2728	#endif
2729	hmR0VmxStructsFree(pVM);
2730	return VINF_SUCCESS;
2731	}
2732
2733
2734	/**
2735	* Sets up the VM for execution under VT-x.
2736	* This function is only called once per-VM during initialization.
2737	*
2738	* @returns VBox status code.
2739	* @param pVM The cross context VM structure.
2740	*/
2741	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2742	{
2743	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2744	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2745
2746	LogFlowFunc(("pVM=%p\n", pVM));
2747
2748	/*
2749	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be
2750	* allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2751	* pRealModeTSS, see hmR3InitFinalizeR0Intel().
2752	*/
2753	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2754	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2755	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2756	{
2757	LogRelFunc(("Invalid real-on-v86 state.\n"));
2758	return VERR_INTERNAL_ERROR;
2759	}
2760
2761	/* Initialize these always, see hmR3InitFinalizeR0().*/
2762	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2763	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2764
2765	/* Setup the tagged-TLB flush handlers. */
2766	int rc = hmR0VmxSetupTaggedTlb(pVM);
2767	if (RT_FAILURE(rc))
2768	{
2769	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2770	return rc;
2771	}
2772
2773	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2774	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2775	#if HC_ARCH_BITS == 64
2776	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2777	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2778	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2779	{
2780	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2781	}
2782	#endif
2783
2784	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2785	RTCCUINTREG uHostCR4 = ASMGetCR4();
2786	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2787	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2788
2789	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2790	{
2791	PVMCPU pVCpu = &pVM->aCpus[i];
2792	AssertPtr(pVCpu);
2793	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2794
2795	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2796	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2797
2798	/* Set revision dword at the beginning of the VMCS structure. */
2799	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2800
2801	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2802	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2803	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2804	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2805
2806	/* Load this VMCS as the current VMCS. */
2807	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2808	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2809	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811	rc = hmR0VmxSetupPinCtls(pVCpu);
2812	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2813	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2814
2815	rc = hmR0VmxSetupProcCtls(pVCpu);
2816	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2817	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2818
2819	rc = hmR0VmxSetupMiscCtls(pVCpu);
2820	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2821	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2822
2823	rc = hmR0VmxInitXcptBitmap(pVCpu);
2824	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2825	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2826
2827	#if HC_ARCH_BITS == 32
2828	rc = hmR0VmxInitVmcsReadCache(pVCpu);
2829	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2830	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2831	#endif
2832
2833	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2834	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2835	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2836	hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2837
2838	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2839
2840	hmR0VmxUpdateErrorRecord(pVCpu, rc);
2841	}
2842
2843	return VINF_SUCCESS;
2844	}
2845
2846
2847	/**
2848	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2849	* the VMCS.
2850	*
2851	* @returns VBox status code.
2852	*/
2853	static int hmR0VmxExportHostControlRegs(void)
2854	{
2855	RTCCUINTREG uReg = ASMGetCR0();
2856	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2857	AssertRCReturn(rc, rc);
2858
2859	uReg = ASMGetCR3();
2860	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2861	AssertRCReturn(rc, rc);
2862
2863	uReg = ASMGetCR4();
2864	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2865	AssertRCReturn(rc, rc);
2866	return rc;
2867	}
2868
2869
2870	/**
2871	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2872	* the host-state area in the VMCS.
2873	*
2874	* @returns VBox status code.
2875	* @param pVCpu The cross context virtual CPU structure.
2876	*/
2877	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2878	{
2879	#if HC_ARCH_BITS == 64
2880	/**
2881	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2882	* requirements. See hmR0VmxExportHostSegmentRegs().
2883	*/
2884	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2885	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2886	{ \
2887	bool fValidSelector = true; \
2888	if ((selValue) & X86_SEL_LDT) \
2889	{ \
2890	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2891	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2892	} \
2893	if (fValidSelector) \
2894	{ \
2895	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2896	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2897	} \
2898	(selValue) = 0; \
2899	}
2900
2901	/*
2902	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2903	* should -not- save the messed up state without restoring the original host-state,
2904	* see @bugref{7240}.
2905	*
2906	* This apparently can happen (most likely the FPU changes), deal with it rather than
2907	* asserting. Was observed booting Solaris 10u10 32-bit guest.
2908	*/
2909	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2910	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2911	{
2912	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2913	pVCpu->idCpu));
2914	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2915	}
2916	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2917	#else
2918	RT_NOREF(pVCpu);
2919	#endif
2920
2921	/*
2922	* Host DS, ES, FS and GS segment registers.
2923	*/
2924	#if HC_ARCH_BITS == 64
2925	RTSEL uSelDS = ASMGetDS();
2926	RTSEL uSelES = ASMGetES();
2927	RTSEL uSelFS = ASMGetFS();
2928	RTSEL uSelGS = ASMGetGS();
2929	#else
2930	RTSEL uSelDS = 0;
2931	RTSEL uSelES = 0;
2932	RTSEL uSelFS = 0;
2933	RTSEL uSelGS = 0;
2934	#endif
2935
2936	/*
2937	* Host CS and SS segment registers.
2938	*/
2939	RTSEL uSelCS = ASMGetCS();
2940	RTSEL uSelSS = ASMGetSS();
2941
2942	/*
2943	* Host TR segment register.
2944	*/
2945	RTSEL uSelTR = ASMGetTR();
2946
2947	#if HC_ARCH_BITS == 64
2948	/*
2949	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2950	* gain VM-entry and restore them before we get preempted.
2951	*
2952	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2953	*/
2954	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2955	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2956	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2957	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2958	# undef VMXLOCAL_ADJUST_HOST_SEG
2959	#endif
2960
2961	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2962	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2963	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2964	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2965	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2966	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2967	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2968	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2969	Assert(uSelCS);
2970	Assert(uSelTR);
2971
2972	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2973	#if 0
2974	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2975	Assert(uSelSS != 0);
2976	#endif
2977
2978	/* Write these host selector fields into the host-state area in the VMCS. */
2979	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2980	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2981	#if HC_ARCH_BITS == 64
2982	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2983	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2984	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2985	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2986	#else
2987	NOREF(uSelDS);
2988	NOREF(uSelES);
2989	NOREF(uSelFS);
2990	NOREF(uSelGS);
2991	#endif
2992	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2993	AssertRCReturn(rc, rc);
2994
2995	/*
2996	* Host GDTR and IDTR.
2997	*/
2998	RTGDTR Gdtr;
2999	RTIDTR Idtr;
3000	RT_ZERO(Gdtr);
3001	RT_ZERO(Idtr);
3002	ASMGetGDTR(&Gdtr);
3003	ASMGetIDTR(&Idtr);
3004	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3005	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3006	AssertRCReturn(rc, rc);
3007
3008	#if HC_ARCH_BITS == 64
3009	/*
3010	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3011	* them to the maximum limit (0xffff) on every VM-exit.
3012	*/
3013	if (Gdtr.cbGdt != 0xffff)
3014	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3015
3016	/*
3017	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3018	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3019	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3020	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3021	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3022	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3023	* at 0xffff on hosts where we are sure it won't cause trouble.
3024	*/
3025	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3026	if (Idtr.cbIdt < 0x0fff)
3027	# else
3028	if (Idtr.cbIdt != 0xffff)
3029	# endif
3030	{
3031	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3032	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3033	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3034	}
3035	#endif
3036
3037	/*
3038	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3039	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3040	* RPL should be too in most cases.
3041	*/
3042	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3043	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3044
3045	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3046	#if HC_ARCH_BITS == 64
3047	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3048
3049	/*
3050	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3051	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3052	* restoration if the host has something else. Task switching is not supported in 64-bit
3053	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3054	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3055	*
3056	* [1] See Intel spec. 3.5 "System Descriptor Types".
3057	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3058	*/
3059	PVM pVM = pVCpu->CTX_SUFF(pVM);
3060	Assert(pDesc->System.u4Type == 11);
3061	if ( pDesc->System.u16LimitLow != 0x67
3062	\|\| pDesc->System.u4LimitHigh)
3063	{
3064	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3065	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3066	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3067	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3068	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3069	}
3070
3071	/*
3072	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3073	*/
3074	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
3075	{
3076	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3077	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3078	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3079	{
3080	/* The GDT is read-only but the writable GDT is available. */
3081	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3082	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3083	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3084	AssertRCReturn(rc, rc);
3085	}
3086	}
3087	#else
3088	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3089	#endif
3090	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3091	AssertRCReturn(rc, rc);
3092
3093	/*
3094	* Host FS base and GS base.
3095	*/
3096	#if HC_ARCH_BITS == 64
3097	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3098	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3099	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3100	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3101	AssertRCReturn(rc, rc);
3102
3103	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3104	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3105	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3106	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3107	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3108	#endif
3109	return VINF_SUCCESS;
3110	}
3111
3112
3113	/**
3114	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
3115	* host-state area of the VMCS.
3116	*
3117	* Theses MSRs will be automatically restored on the host after every successful
3118	* VM-exit.
3119	*
3120	* @returns VBox status code.
3121	* @param pVCpu The cross context virtual CPU structure.
3122	*
3123	* @remarks No-long-jump zone!!!
3124	*/
3125	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3126	{
3127	AssertPtr(pVCpu);
3128	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3129
3130	/*
3131	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3132	* rather than swapping them on every VM-entry.
3133	*/
3134	hmR0VmxLazySaveHostMsrs(pVCpu);
3135
3136	/*
3137	* Host Sysenter MSRs.
3138	*/
3139	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3140	#if HC_ARCH_BITS == 32
3141	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3142	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3143	#else
3144	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3145	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3146	#endif
3147	AssertRCReturn(rc, rc);
3148
3149	/*
3150	* Host EFER MSR.
3151	*
3152	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3153	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3154	*/
3155	PVM pVM = pVCpu->CTX_SUFF(pVM);
3156	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3157	{
3158	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3159	AssertRCReturn(rc, rc);
3160	}
3161
3162	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3163
3164	return VINF_SUCCESS;
3165	}
3166
3167
3168	/**
3169	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3170	*
3171	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3172	* these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3173	* hmR0VMxExportGuestEntryCtls().
3174	*
3175	* @returns true if we need to load guest EFER, false otherwise.
3176	* @param pVCpu The cross context virtual CPU structure.
3177	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3178	* out-of-sync. Make sure to update the required fields
3179	* before using them.
3180	*
3181	* @remarks Requires EFER, CR4.
3182	* @remarks No-long-jump zone!!!
3183	*/
3184	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3185	{
3186	#ifdef HMVMX_ALWAYS_SWAP_EFER
3187	return true;
3188	#endif
3189
3190	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3191	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3192	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3193	return false;
3194	#endif
3195
3196	PVM pVM = pVCpu->CTX_SUFF(pVM);
3197	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3198	uint64_t const u64GuestEfer = pMixedCtx->msrEFER;
3199
3200	/*
3201	* For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3202	* guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3203	*/
3204	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3205	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3206	{
3207	return true;
3208	}
3209
3210	/*
3211	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3212	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3213	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3214	*/
3215	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3216	&& (pMixedCtx->cr0 & X86_CR0_PG)
3217	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3218	{
3219	/* Assert that host is PAE capable. */
3220	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3221	return true;
3222	}
3223
3224	return false;
3225	}
3226
3227
3228	/**
3229	* Exports the guest state with appropriate VM-entry controls in the VMCS.
3230	*
3231	* These controls can affect things done on VM-exit; e.g. "load debug controls",
3232	* see Intel spec. 24.8.1 "VM-entry controls".
3233	*
3234	* @returns VBox status code.
3235	* @param pVCpu The cross context virtual CPU structure.
3236	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3237	* out-of-sync. Make sure to update the required fields
3238	* before using them.
3239	*
3240	* @remarks Requires EFER.
3241	* @remarks No-long-jump zone!!!
3242	*/
3243	static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3244	{
3245	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3246	{
3247	PVM pVM = pVCpu->CTX_SUFF(pVM);
3248	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3249	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3250
3251	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3252	fVal \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3253
3254	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3255	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3256	{
3257	fVal \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3258	Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3259	}
3260	else
3261	Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3262
3263	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3264	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3265	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3266	{
3267	fVal \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3268	Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3269	}
3270
3271	/*
3272	* The following should -not- be set (since we're not in SMM mode):
3273	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275	*/
3276
3277	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280	if ((fVal & fZap) != fVal)
3281	{
3282	Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3283	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3284	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286	}
3287
3288	/* Commit it to the VMCS and update our cache. */
3289	if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3290	{
3291	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3292	AssertRCReturn(rc, rc);
3293	pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3294	}
3295
3296	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3297	}
3298	return VINF_SUCCESS;
3299	}
3300
3301
3302	/**
3303	* Exports the guest state with appropriate VM-exit controls in the VMCS.
3304	*
3305	* @returns VBox status code.
3306	* @param pVCpu The cross context virtual CPU structure.
3307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3308	* out-of-sync. Make sure to update the required fields
3309	* before using them.
3310	*
3311	* @remarks Requires EFER.
3312	*/
3313	static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3314	{
3315	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3316	{
3317	PVM pVM = pVCpu->CTX_SUFF(pVM);
3318	uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3319	uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3320
3321	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3322	fVal \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3323
3324	/*
3325	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3326	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3327	* hmR0VmxExportHostMsrs().
3328	*/
3329	#if HC_ARCH_BITS == 64
3330	fVal \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3331	Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3332	#else
3333	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3334	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3335	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3336	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3337	{
3338	/* The switcher returns to long mode, EFER is managed by the switcher. */
3339	fVal \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3340	Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3341	}
3342	else
3343	Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3344	#endif
3345
3346	/* If the newer VMCS fields for managing EFER exists, use it. */
3347	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3348	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3349	{
3350	fVal \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3351	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3352	Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3353	}
3354
3355	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3356	Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3357
3358	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3359	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3360	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3361
3362	/* Enable saving of the VMX preemption timer value on VM-exit. */
3363	if ( pVM->hm.s.vmx.fUsePreemptTimer
3364	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3365	fVal \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3366
3367	if ((fVal & fZap) != fVal)
3368	{
3369	LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3370	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3371	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3372	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3373	}
3374
3375	/* Commit it to the VMCS and update our cache. */
3376	if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3377	{
3378	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3379	AssertRCReturn(rc, rc);
3380	pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3381	}
3382
3383	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3384	}
3385	return VINF_SUCCESS;
3386	}
3387
3388
3389	/**
3390	* Sets the TPR threshold in the VMCS.
3391	*
3392	* @returns VBox status code.
3393	* @param pVCpu The cross context virtual CPU structure.
3394	* @param u32TprThreshold The TPR threshold (task-priority class only).
3395	*/
3396	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3397	{
3398	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3399	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3400	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3401	}
3402
3403
3404	/**
3405	* Exports the guest APIC TPR state into the VMCS.
3406	*
3407	* @returns VBox status code.
3408	* @param pVCpu The cross context virtual CPU structure.
3409	*
3410	* @remarks No-long-jump zone!!!
3411	*/
3412	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3413	{
3414	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3415	{
3416	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3417
3418	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3419	&& APICIsEnabled(pVCpu))
3420	{
3421	/*
3422	* Setup TPR shadowing.
3423	*/
3424	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3425	{
3426	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3427
3428	bool fPendingIntr = false;
3429	uint8_t u8Tpr = 0;
3430	uint8_t u8PendingIntr = 0;
3431	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3432	AssertRCReturn(rc, rc);
3433
3434	/*
3435	* If there are interrupts pending but masked by the TPR, instruct VT-x to
3436	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3437	* priority of the pending interrupt so we can deliver the interrupt. If there
3438	* are no interrupts pending, set threshold to 0 to not cause any
3439	* TPR-below-threshold VM-exits.
3440	*/
3441	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3442	uint32_t u32TprThreshold = 0;
3443	if (fPendingIntr)
3444	{
3445	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3446	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3447	const uint8_t u8TprPriority = u8Tpr >> 4;
3448	if (u8PendingPriority <= u8TprPriority)
3449	u32TprThreshold = u8PendingPriority;
3450	}
3451
3452	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3453	AssertRCReturn(rc, rc);
3454	}
3455	}
3456	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3457	}
3458	return VINF_SUCCESS;
3459	}
3460
3461
3462	/**
3463	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3464	*
3465	* @returns Guest's interruptibility-state.
3466	* @param pVCpu The cross context virtual CPU structure.
3467	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3468	* out-of-sync. Make sure to update the required fields
3469	* before using them.
3470	*
3471	* @remarks No-long-jump zone!!!
3472	*/
3473	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3474	{
3475	/*
3476	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3477	*/
3478	uint32_t fIntrState = 0;
3479	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3480	{
3481	/* If inhibition is active, RIP & RFLAGS should've been accessed
3482	(i.e. read previously from the VMCS or from ring-3). */
3483	#ifdef VBOX_STRICT
3484	uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3485	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3486	#endif
3487	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3488	{
3489	if (pMixedCtx->eflags.Bits.u1IF)
3490	fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3491	else
3492	fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3493	}
3494	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3495	{
3496	/*
3497	* We can clear the inhibit force flag as even if we go back to the recompiler
3498	* without executing guest code in VT-x, the flag's condition to be cleared is
3499	* met and thus the cleared state is correct.
3500	*/
3501	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3502	}
3503	}
3504
3505	/*
3506	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3507	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3508	* setting this would block host-NMIs and IRET will not clear the blocking.
3509	*
3510	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3511	*/
3512	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3513	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3514	{
3515	fIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3516	}
3517
3518	return fIntrState;
3519	}
3520
3521
3522	/**
3523	* Exports the guest's interruptibility-state into the guest-state area in the
3524	* VMCS.
3525	*
3526	* @returns VBox status code.
3527	* @param pVCpu The cross context virtual CPU structure.
3528	* @param fIntrState The interruptibility-state to set.
3529	*/
3530	static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3531	{
3532	NOREF(pVCpu);
3533	AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3534	Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3535	return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3536	}
3537
3538
3539	/**
3540	* Exports the exception intercepts required for guest execution in the VMCS.
3541	*
3542	* @returns VBox status code.
3543	* @param pVCpu The cross context virtual CPU structure.
3544	*
3545	* @remarks No-long-jump zone!!!
3546	*/
3547	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3548	{
3549	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3550	{
3551	uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3552
3553	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3554	if (pVCpu->hm.s.fGIMTrapXcptUD)
3555	uXcptBitmap \|= RT_BIT(X86_XCPT_UD);
3556	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3557	else
3558	uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3559	#endif
3560
3561	Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3562	Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3563
3564	if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3565	{
3566	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3567	AssertRCReturn(rc, rc);
3568	pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3569	}
3570
3571	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3572	Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3573	}
3574	return VINF_SUCCESS;
3575	}
3576
3577
3578	/**
3579	* Exports the guest's RIP into the guest-state area in the VMCS.
3580	*
3581	* @returns VBox status code.
3582	* @param pVCpu The cross context virtual CPU structure.
3583	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3584	* out-of-sync. Make sure to update the required fields
3585	* before using them.
3586	*
3587	* @remarks No-long-jump zone!!!
3588	*/
3589	static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3590	{
3591	int rc = VINF_SUCCESS;
3592	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3593	{
3594	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3595
3596	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3597	AssertRCReturn(rc, rc);
3598
3599	/* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3600	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3601	EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3602	else
3603	EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3604
3605	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3606	Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3607	}
3608	return rc;
3609	}
3610
3611
3612	/**
3613	* Exports the guest's RSP into the guest-state area in the VMCS.
3614	*
3615	* @returns VBox status code.
3616	* @param pVCpu The cross context virtual CPU structure.
3617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3618	* out-of-sync. Make sure to update the required fields
3619	* before using them.
3620	*
3621	* @remarks No-long-jump zone!!!
3622	*/
3623	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3624	{
3625	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3626	{
3627	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3628
3629	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3630	AssertRCReturn(rc, rc);
3631
3632	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3633	}
3634	return VINF_SUCCESS;
3635	}
3636
3637
3638	/**
3639	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
3640	*
3641	* @returns VBox status code.
3642	* @param pVCpu The cross context virtual CPU structure.
3643	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3644	* out-of-sync. Make sure to update the required fields
3645	* before using them.
3646	*
3647	* @remarks No-long-jump zone!!!
3648	*/
3649	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3650	{
3651	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3652	{
3653	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3654
3655	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3656	Let us assert it as such and use 32-bit VMWRITE. */
3657	Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3658	X86EFLAGS fEFlags = pMixedCtx->eflags;
3659	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3660	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3661
3662	/*
3663	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3664	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3665	* can run the real-mode guest code under Virtual 8086 mode.
3666	*/
3667	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3668	{
3669	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3670	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3671	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3672	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3673	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3674	}
3675
3676	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3677	AssertRCReturn(rc, rc);
3678
3679	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3680	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3681	}
3682	return VINF_SUCCESS;
3683	}
3684
3685
3686	/**
3687	* Exports the guest CR0 control register into the guest-state area in the VMCS.
3688	*
3689	* The guest FPU state is always pre-loaded hence we don't need to bother about
3690	* sharing FPU related CR0 bits between the guest and host.
3691	*
3692	* @returns VBox status code.
3693	* @param pVCpu The cross context virtual CPU structure.
3694	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3695	* out-of-sync. Make sure to update the required fields
3696	* before using them.
3697	*
3698	* @remarks No-long-jump zone!!!
3699	*/
3700	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3701	{
3702	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3703	{
3704	PVM pVM = pVCpu->CTX_SUFF(pVM);
3705	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3706	Assert(!RT_HI_U32(pMixedCtx->cr0));
3707
3708	uint32_t const u32ShadowCr0 = pMixedCtx->cr0;
3709	uint32_t u32GuestCr0 = pMixedCtx->cr0;
3710
3711	/*
3712	* Setup VT-x's view of the guest CR0.
3713	* Minimize VM-exits due to CR3 changes when we have NestedPaging.
3714	*/
3715	uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3716	if (pVM->hm.s.fNestedPaging)
3717	{
3718	if (CPUMIsGuestPagingEnabled(pVCpu))
3719	{
3720	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721	uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723	}
3724	else
3725	{
3726	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727	uProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729	}
3730
3731	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733	uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734	}
3735	else
3736	{
3737	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3738	u32GuestCr0 \|= X86_CR0_WP;
3739	}
3740
3741	/*
3742	* Guest FPU bits.
3743	*
3744	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3745	* using CR0.TS.
3746	*
3747	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3748	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3749	*/
3750	u32GuestCr0 \|= X86_CR0_NE;
3751
3752	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3753	bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3754
3755	/*
3756	* Update exception intercepts.
3757	*/
3758	uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3759	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3760	{
3761	Assert(PDMVmmDevHeapIsEnabled(pVM));
3762	Assert(pVM->hm.s.vmx.pRealModeTSS);
3763	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3764	}
3765	else
3766	{
3767	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3768	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3769	if (fInterceptMF)
3770	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
3771	}
3772
3773	/* Additional intercepts for debugging, define these yourself explicitly. */
3774	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3775	uXcptBitmap \|= 0
3776	\| RT_BIT(X86_XCPT_BP)
3777	\| RT_BIT(X86_XCPT_DE)
3778	\| RT_BIT(X86_XCPT_NM)
3779	\| RT_BIT(X86_XCPT_TS)
3780	\| RT_BIT(X86_XCPT_UD)
3781	\| RT_BIT(X86_XCPT_NP)
3782	\| RT_BIT(X86_XCPT_SS)
3783	\| RT_BIT(X86_XCPT_GP)
3784	\| RT_BIT(X86_XCPT_PF)
3785	\| RT_BIT(X86_XCPT_MF)
3786	;
3787	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3788	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
3789	#endif
3790	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3791
3792	/*
3793	* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3794	*/
3795	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3796	uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3797	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3798	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3799	else
3800	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3801
3802	u32GuestCr0 \|= fSetCr0;
3803	u32GuestCr0 &= fZapCr0;
3804	u32GuestCr0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3805
3806	/*
3807	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3808	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3809	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3810	*/
3811	uint32_t u32Cr0Mask = X86_CR0_PE
3812	\| X86_CR0_NE
3813	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3814	\| X86_CR0_PG
3815	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3816	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3817	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3818
3819	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3820	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3821	* and @bugref{6944}. */
3822	#if 0
3823	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3824	u32Cr0Mask &= ~X86_CR0_PE;
3825	#endif
3826	/*
3827	* Finally, update VMCS fields with the CR0 values and the exception bitmap.
3828	*/
3829	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3830	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3831	if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3832	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3833	if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3834	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3835	if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3836	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3837	AssertRCReturn(rc, rc);
3838
3839	/* Update our caches. */
3840	pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3841	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3842	pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3843
3844	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3845
3846	Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3847	u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3848	}
3849
3850	return VINF_SUCCESS;
3851	}
3852
3853
3854	/**
3855	* Exports the guest control registers (CR3, CR4) into the guest-state area
3856	* in the VMCS.
3857	*
3858	* @returns VBox strict status code.
3859	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3860	* without unrestricted guest access and the VMMDev is not presently
3861	* mapped (e.g. EFI32).
3862	*
3863	* @param pVCpu The cross context virtual CPU structure.
3864	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3865	* out-of-sync. Make sure to update the required fields
3866	* before using them.
3867	*
3868	* @remarks No-long-jump zone!!!
3869	*/
3870	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3871	{
3872	int rc = VINF_SUCCESS;
3873	PVM pVM = pVCpu->CTX_SUFF(pVM);
3874
3875	/*
3876	* Guest CR2.
3877	* It's always loaded in the assembler code. Nothing to do here.
3878	*/
3879
3880	/*
3881	* Guest CR3.
3882	*/
3883	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3884	{
3885	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3886
3887	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3888	if (pVM->hm.s.fNestedPaging)
3889	{
3890	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3891
3892	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3893	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3894	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3895	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3896
3897	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3898	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3899	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3900
3901	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3902	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3903	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3904	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3905	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3906	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3907	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3908
3909	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3910	AssertRCReturn(rc, rc);
3911
3912	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3913	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3914	{
3915	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3916	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3917	{
3918	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3919	AssertRCReturn(rc, rc);
3920	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3921	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3922	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3923	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3924	AssertRCReturn(rc, rc);
3925	}
3926
3927	/*
3928	* The guest's view of its CR3 is unblemished with Nested Paging when the
3929	* guest is using paging or we have unrestricted guest execution to handle
3930	* the guest when it's not using paging.
3931	*/
3932	GCPhysGuestCR3 = pMixedCtx->cr3;
3933	}
3934	else
3935	{
3936	/*
3937	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
3938	* thinks it accesses physical memory directly, we use our identity-mapped
3939	* page table to map guest-linear to guest-physical addresses. EPT takes care
3940	* of translating it to host-physical addresses.
3941	*/
3942	RTGCPHYS GCPhys;
3943	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3944
3945	/* We obtain it here every time as the guest could have relocated this PCI region. */
3946	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3947	if (RT_SUCCESS(rc))
3948	{ /* likely */ }
3949	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3950	{
3951	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3952	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3953	}
3954	else
3955	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3956
3957	GCPhysGuestCR3 = GCPhys;
3958	}
3959
3960	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3961	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3962	AssertRCReturn(rc, rc);
3963	}
3964	else
3965	{
3966	/* Non-nested paging case, just use the hypervisor's CR3. */
3967	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3968
3969	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3970	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3971	AssertRCReturn(rc, rc);
3972	}
3973
3974	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3975	}
3976
3977	/*
3978	* Guest CR4.
3979	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
3980	*/
3981	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3982	{
3983	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3984	Assert(!RT_HI_U32(pMixedCtx->cr4));
3985
3986	uint32_t u32GuestCr4 = pMixedCtx->cr4;
3987	uint32_t const u32ShadowCr4 = pMixedCtx->cr4;
3988
3989	/*
3990	* Setup VT-x's view of the guest CR4.
3991	*
3992	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3993	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3994	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3995	*
3996	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3997	*/
3998	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3999	{
4000	Assert(pVM->hm.s.vmx.pRealModeTSS);
4001	Assert(PDMVmmDevHeapIsEnabled(pVM));
4002	u32GuestCr4 &= ~X86_CR4_VME;
4003	}
4004
4005	if (pVM->hm.s.fNestedPaging)
4006	{
4007	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4008	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4009	{
4010	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4011	u32GuestCr4 \|= X86_CR4_PSE;
4012	/* Our identity mapping is a 32-bit page directory. */
4013	u32GuestCr4 &= ~X86_CR4_PAE;
4014	}
4015	/* else use guest CR4.*/
4016	}
4017	else
4018	{
4019	/*
4020	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4021	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4022	*/
4023	switch (pVCpu->hm.s.enmShadowMode)
4024	{
4025	case PGMMODE_REAL: /* Real-mode. */
4026	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4027	case PGMMODE_32_BIT: /* 32-bit paging. */
4028	{
4029	u32GuestCr4 &= ~X86_CR4_PAE;
4030	break;
4031	}
4032
4033	case PGMMODE_PAE: /* PAE paging. */
4034	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4035	{
4036	u32GuestCr4 \|= X86_CR4_PAE;
4037	break;
4038	}
4039
4040	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4041	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4042	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4043	break;
4044	#endif
4045	default:
4046	AssertFailed();
4047	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4048	}
4049	}
4050
4051	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4052	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4053	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4054	u32GuestCr4 \|= fSetCr4;
4055	u32GuestCr4 &= fZapCr4;
4056
4057	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4058	that would cause a VM-exit. */
4059	uint32_t u32Cr4Mask = X86_CR4_VME
4060	\| X86_CR4_PAE
4061	\| X86_CR4_PGE
4062	\| X86_CR4_PSE
4063	\| X86_CR4_VMXE;
4064	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4065	u32Cr4Mask \|= X86_CR4_OSXSAVE;
4066	if (pVM->cpum.ro.GuestFeatures.fPcid)
4067	u32Cr4Mask \|= X86_CR4_PCIDE;
4068
4069	/* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4070	into the VMCS and update our cache. */
4071	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4072	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4073	if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4074	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4075	AssertRCReturn(rc, rc);
4076	pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4077
4078	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4079	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4080
4081	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4082
4083	Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4084	fZapCr4));
4085	}
4086	return rc;
4087	}
4088
4089
4090	/**
4091	* Exports the guest debug registers into the guest-state area in the VMCS.
4092	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4093	*
4094	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4095	*
4096	* @returns VBox status code.
4097	* @param pVCpu The cross context virtual CPU structure.
4098	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4099	* out-of-sync. Make sure to update the required fields
4100	* before using them.
4101	*
4102	* @remarks No-long-jump zone!!!
4103	*/
4104	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4105	{
4106	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4107
4108	#ifdef VBOX_STRICT
4109	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4110	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4111	{
4112	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4113	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4114	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4115	}
4116	#endif
4117
4118	bool fSteppingDB = false;
4119	bool fInterceptMovDRx = false;
4120	uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4121	if (pVCpu->hm.s.fSingleInstruction)
4122	{
4123	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4124	PVM pVM = pVCpu->CTX_SUFF(pVM);
4125	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4126	{
4127	uProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4128	Assert(fSteppingDB == false);
4129	}
4130	else
4131	{
4132	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4133	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
4134	pVCpu->hm.s.fClearTrapFlag = true;
4135	fSteppingDB = true;
4136	}
4137	}
4138
4139	uint32_t u32GuestDr7;
4140	if ( fSteppingDB
4141	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4142	{
4143	/*
4144	* Use the combined guest and host DRx values found in the hypervisor register set
4145	* because the debugger has breakpoints active or someone is single stepping on the
4146	* host side without a monitor trap flag.
4147	*
4148	* Note! DBGF expects a clean DR6 state before executing guest code.
4149	*/
4150	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4151	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4152	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4153	{
4154	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4155	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4156	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4157	}
4158	else
4159	#endif
4160	if (!CPUMIsHyperDebugStateActive(pVCpu))
4161	{
4162	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4163	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4164	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4165	}
4166
4167	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4168	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4169	pVCpu->hm.s.fUsingHyperDR7 = true;
4170	fInterceptMovDRx = true;
4171	}
4172	else
4173	{
4174	/*
4175	* If the guest has enabled debug registers, we need to load them prior to
4176	* executing guest code so they'll trigger at the right time.
4177	*/
4178	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
4179	{
4180	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4181	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4182	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4183	{
4184	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4185	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4186	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4187	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4188	}
4189	else
4190	#endif
4191	if (!CPUMIsGuestDebugStateActive(pVCpu))
4192	{
4193	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4194	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4195	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4196	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4197	}
4198	Assert(!fInterceptMovDRx);
4199	}
4200	/*
4201	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4202	* must intercept #DB in order to maintain a correct DR6 guest value, and
4203	* because we need to intercept it to prevent nested #DBs from hanging the
4204	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4205	*/
4206	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4207	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4208	&& !CPUMIsGuestDebugStateActive(pVCpu))
4209	#else
4210	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4211	#endif
4212	{
4213	fInterceptMovDRx = true;
4214	}
4215
4216	/* Update DR7 with the actual guest value. */
4217	u32GuestDr7 = pMixedCtx->dr[7];
4218	pVCpu->hm.s.fUsingHyperDR7 = false;
4219	}
4220
4221	if (fInterceptMovDRx)
4222	uProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4223	else
4224	uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4225
4226	/*
4227	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4228	* monitor-trap flag and update our cache.
4229	*/
4230	if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4231	{
4232	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4233	AssertRCReturn(rc2, rc2);
4234	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4235	}
4236
4237	/*
4238	* Update guest DR7.
4239	*/
4240	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4241	AssertRCReturn(rc, rc);
4242
4243	return VINF_SUCCESS;
4244	}
4245
4246
4247	#ifdef VBOX_STRICT
4248	/**
4249	* Strict function to validate segment registers.
4250	*
4251	* @param pVCpu The cross context virtual CPU structure.
4252	* @param pCtx Pointer to the guest-CPU context.
4253	*
4254	* @remarks Will import guest CR0 on strict builds during validation of
4255	* segments.
4256	*/
4257	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4258	{
4259	/*
4260	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4261	*
4262	* The reason we check for attribute value 0 in this function and not just the unusable bit is
4263	* because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4264	* and doesn't change the guest-context value.
4265	*/
4266	PVM pVM = pVCpu->CTX_SUFF(pVM);
4267	hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4268	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4269	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4270	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4271	{
4272	/* Protected mode checks */
4273	/* CS */
4274	Assert(pCtx->cs.Attr.n.u1Present);
4275	Assert(!(pCtx->cs.Attr.u & 0xf00));
4276	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4277	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4278	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4279	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4280	\|\| (pCtx->cs.Attr.n.u1Granularity));
4281	/* CS cannot be loaded with NULL in protected mode. */
4282	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4283	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4284	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4285	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4286	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4287	else
4288	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4289	/* SS */
4290	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4291	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4292	if ( !(pCtx->cr0 & X86_CR0_PE)
4293	\|\| pCtx->cs.Attr.n.u4Type == 3)
4294	{
4295	Assert(!pCtx->ss.Attr.n.u2Dpl);
4296	}
4297	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4298	{
4299	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4300	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4301	Assert(pCtx->ss.Attr.n.u1Present);
4302	Assert(!(pCtx->ss.Attr.u & 0xf00));
4303	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4304	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4305	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4306	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4307	\|\| (pCtx->ss.Attr.n.u1Granularity));
4308	}
4309	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4310	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4311	{
4312	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4313	Assert(pCtx->ds.Attr.n.u1Present);
4314	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4315	Assert(!(pCtx->ds.Attr.u & 0xf00));
4316	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4317	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4318	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4319	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4320	\|\| (pCtx->ds.Attr.n.u1Granularity));
4321	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4322	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4323	}
4324	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4325	{
4326	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4327	Assert(pCtx->es.Attr.n.u1Present);
4328	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4329	Assert(!(pCtx->es.Attr.u & 0xf00));
4330	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4331	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4332	\|\| !(pCtx->es.Attr.n.u1Granularity));
4333	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4334	\|\| (pCtx->es.Attr.n.u1Granularity));
4335	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4336	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4337	}
4338	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4339	{
4340	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4341	Assert(pCtx->fs.Attr.n.u1Present);
4342	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4343	Assert(!(pCtx->fs.Attr.u & 0xf00));
4344	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4345	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4346	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4347	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4348	\|\| (pCtx->fs.Attr.n.u1Granularity));
4349	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4350	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4351	}
4352	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4353	{
4354	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4355	Assert(pCtx->gs.Attr.n.u1Present);
4356	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4357	Assert(!(pCtx->gs.Attr.u & 0xf00));
4358	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4359	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4360	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4361	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4362	\|\| (pCtx->gs.Attr.n.u1Granularity));
4363	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4364	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4365	}
4366	/* 64-bit capable CPUs. */
4367	# if HC_ARCH_BITS == 64
4368	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4369	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
4370	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
4371	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
4372	# endif
4373	}
4374	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4375	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4376	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4377	{
4378	/* Real and v86 mode checks. */
4379	/* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4380	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4381	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4382	{
4383	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4384	}
4385	else
4386	{
4387	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4388	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4389	}
4390
4391	/* CS */
4392	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4393	Assert(pCtx->cs.u32Limit == 0xffff);
4394	Assert(u32CSAttr == 0xf3);
4395	/* SS */
4396	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4397	Assert(pCtx->ss.u32Limit == 0xffff);
4398	Assert(u32SSAttr == 0xf3);
4399	/* DS */
4400	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4401	Assert(pCtx->ds.u32Limit == 0xffff);
4402	Assert(u32DSAttr == 0xf3);
4403	/* ES */
4404	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4405	Assert(pCtx->es.u32Limit == 0xffff);
4406	Assert(u32ESAttr == 0xf3);
4407	/* FS */
4408	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4409	Assert(pCtx->fs.u32Limit == 0xffff);
4410	Assert(u32FSAttr == 0xf3);
4411	/* GS */
4412	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4413	Assert(pCtx->gs.u32Limit == 0xffff);
4414	Assert(u32GSAttr == 0xf3);
4415	/* 64-bit capable CPUs. */
4416	# if HC_ARCH_BITS == 64
4417	Assert(!RT_HI_U32(pCtx->cs.u64Base));
4418	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
4419	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
4420	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
4421	# endif
4422	}
4423	}
4424	#endif /* VBOX_STRICT */
4425
4426
4427	/**
4428	* Exports a guest segment register into the guest-state area in the VMCS.
4429	*
4430	* @returns VBox status code.
4431	* @param pVCpu The cross context virtual CPU structure.
4432	* @param idxSel Index of the selector in the VMCS.
4433	* @param idxLimit Index of the segment limit in the VMCS.
4434	* @param idxBase Index of the segment base in the VMCS.
4435	* @param idxAccess Index of the access rights of the segment in the VMCS.
4436	* @param pSelReg Pointer to the segment selector.
4437	*
4438	* @remarks No-long-jump zone!!!
4439	*/
4440	static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4441	PCCPUMSELREG pSelReg)
4442	{
4443	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4444	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4445	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4446	AssertRCReturn(rc, rc);
4447
4448	uint32_t u32Access = pSelReg->Attr.u;
4449	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4450	{
4451	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4452	u32Access = 0xf3;
4453	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4454	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4455	}
4456	else
4457	{
4458	/*
4459	* The way to differentiate between whether this is really a null selector or was just
4460	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
4461	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4462	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4463	* NULL selectors loaded in protected-mode have their attribute as 0.
4464	*/
4465	if (!u32Access)
4466	u32Access = X86DESCATTR_UNUSABLE;
4467	}
4468
4469	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4470	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4471	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4472
4473	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4474	AssertRCReturn(rc, rc);
4475	return rc;
4476	}
4477
4478
4479	/**
4480	* Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4481	* into the guest-state area in the VMCS.
4482	*
4483	* @returns VBox status code.
4484	* @param pVCpu The cross context virtual CPU structure.
4485	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4486	* out-of-sync. Make sure to update the required fields
4487	* before using them.
4488	*
4489	* @remarks Will import guest CR0 on strict builds during validation of
4490	* segments.
4491	* @remarks No-long-jump zone!!!
4492	*/
4493	static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4494	{
4495	int rc = VERR_INTERNAL_ERROR_5;
4496	PVM pVM = pVCpu->CTX_SUFF(pVM);
4497
4498	/*
4499	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4500	*/
4501	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4502	{
4503	#ifdef VBOX_WITH_REM
4504	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4505	{
4506	Assert(pVM->hm.s.vmx.pRealModeTSS);
4507	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4508	if ( pVCpu->hm.s.vmx.fWasInRealMode
4509	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4510	{
4511	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4512	in real-mode (e.g. OpenBSD 4.0) */
4513	REMFlushTBs(pVM);
4514	Log4Func(("Switch to protected mode detected!\n"));
4515	pVCpu->hm.s.vmx.fWasInRealMode = false;
4516	}
4517	}
4518	#endif
4519	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4520	{
4521	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4522	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4523	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4524	rc = HMVMX_EXPORT_SREG(CS, &pMixedCtx->cs);
4525	AssertRCReturn(rc, rc);
4526	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4527	}
4528
4529	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4530	{
4531	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4532	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4533	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4534	rc = HMVMX_EXPORT_SREG(SS, &pMixedCtx->ss);
4535	AssertRCReturn(rc, rc);
4536	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4537	}
4538
4539	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4540	{
4541	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4542	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4543	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4544	rc = HMVMX_EXPORT_SREG(DS, &pMixedCtx->ds);
4545	AssertRCReturn(rc, rc);
4546	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4547	}
4548
4549	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4550	{
4551	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4552	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4553	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4554	rc = HMVMX_EXPORT_SREG(ES, &pMixedCtx->es);
4555	AssertRCReturn(rc, rc);
4556	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4557	}
4558
4559	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4560	{
4561	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4562	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4563	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4564	rc = HMVMX_EXPORT_SREG(FS, &pMixedCtx->fs);
4565	AssertRCReturn(rc, rc);
4566	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4567	}
4568
4569	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4570	{
4571	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4572	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4573	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4574	rc = HMVMX_EXPORT_SREG(GS, &pMixedCtx->gs);
4575	AssertRCReturn(rc, rc);
4576	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4577	}
4578
4579	#ifdef VBOX_STRICT
4580	hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4581	#endif
4582
4583	/* Update the exit history entry with the correct CS.BASE + RIP. */
4584	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4585	EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4586
4587	Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4588	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4589	}
4590
4591	/*
4592	* Guest TR.
4593	*/
4594	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4595	{
4596	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4597
4598	/*
4599	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4600	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4601	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4602	*/
4603	uint16_t u16Sel = 0;
4604	uint32_t u32Limit = 0;
4605	uint64_t u64Base = 0;
4606	uint32_t u32AccessRights = 0;
4607
4608	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4609	{
4610	u16Sel = pMixedCtx->tr.Sel;
4611	u32Limit = pMixedCtx->tr.u32Limit;
4612	u64Base = pMixedCtx->tr.u64Base;
4613	u32AccessRights = pMixedCtx->tr.Attr.u;
4614	}
4615	else
4616	{
4617	Assert(pVM->hm.s.vmx.pRealModeTSS);
4618	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4619
4620	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4621	RTGCPHYS GCPhys;
4622	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4623	AssertRCReturn(rc, rc);
4624
4625	X86DESCATTR DescAttr;
4626	DescAttr.u = 0;
4627	DescAttr.n.u1Present = 1;
4628	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4629
4630	u16Sel = 0;
4631	u32Limit = HM_VTX_TSS_SIZE;
4632	u64Base = GCPhys; /* in real-mode phys = virt. */
4633	u32AccessRights = DescAttr.u;
4634	}
4635
4636	/* Validate. */
4637	Assert(!(u16Sel & RT_BIT(2)));
4638	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4639	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4640	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4641	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4642	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4643	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4644	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4645	Assert( (u32Limit & 0xfff) == 0xfff
4646	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4647	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4648	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4649
4650	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4651	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4652	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4653	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4654	AssertRCReturn(rc, rc);
4655
4656	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4657	Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4658	}
4659
4660	/*
4661	* Guest GDTR.
4662	*/
4663	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4664	{
4665	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4666
4667	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4668	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4669	AssertRCReturn(rc, rc);
4670
4671	/* Validate. */
4672	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4673
4674	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4675	Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4676	}
4677
4678	/*
4679	* Guest LDTR.
4680	*/
4681	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4682	{
4683	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4684
4685	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4686	uint32_t u32Access = 0;
4687	if (!pMixedCtx->ldtr.Attr.u)
4688	u32Access = X86DESCATTR_UNUSABLE;
4689	else
4690	u32Access = pMixedCtx->ldtr.Attr.u;
4691
4692	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4693	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4694	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4695	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4696	AssertRCReturn(rc, rc);
4697
4698	/* Validate. */
4699	if (!(u32Access & X86DESCATTR_UNUSABLE))
4700	{
4701	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4702	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4703	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4704	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4705	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4706	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4707	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4708	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4709	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4710	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4711	}
4712
4713	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4714	Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4715	}
4716
4717	/*
4718	* Guest IDTR.
4719	*/
4720	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4721	{
4722	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4723
4724	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4725	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4726	AssertRCReturn(rc, rc);
4727
4728	/* Validate. */
4729	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4730
4731	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4732	Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4733	}
4734
4735	return VINF_SUCCESS;
4736	}
4737
4738
4739	/**
4740	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4741	* areas.
4742	*
4743	* These MSRs will automatically be loaded to the host CPU on every successful
4744	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4745	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4746	* -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4747	*
4748	* Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4749	*
4750	* @returns VBox status code.
4751	* @param pVCpu The cross context virtual CPU structure.
4752	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4753	* out-of-sync. Make sure to update the required fields
4754	* before using them.
4755	*
4756	* @remarks No-long-jump zone!!!
4757	*/
4758	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4759	{
4760	AssertPtr(pVCpu);
4761	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4762
4763	/*
4764	* MSRs that we use the auto-load/store MSR area in the VMCS.
4765	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4766	*/
4767	PVM pVM = pVCpu->CTX_SUFF(pVM);
4768	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4769	{
4770	if (pVM->hm.s.fAllow64BitGuests)
4771	{
4772	#if HC_ARCH_BITS == 32
4773	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
4774
4775	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4776	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4777	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4778	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4779	AssertRCReturn(rc, rc);
4780	# ifdef LOG_ENABLED
4781	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4782	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4783	Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4784	# endif
4785	#endif
4786	}
4787	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4788	}
4789
4790	/*
4791	* Guest Sysenter MSRs.
4792	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4793	* VM-exits on WRMSRs for these MSRs.
4794	*/
4795	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4796	{
4797	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4798
4799	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4800	{
4801	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4802	AssertRCReturn(rc, rc);
4803	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4804	}
4805
4806	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4807	{
4808	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4809	AssertRCReturn(rc, rc);
4810	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4811	}
4812
4813	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4814	{
4815	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4816	AssertRCReturn(rc, rc);
4817	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4818	}
4819	}
4820
4821	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4822	{
4823	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4824
4825	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4826	{
4827	/*
4828	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4829	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4830	*/
4831	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4832	{
4833	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4834	AssertRCReturn(rc,rc);
4835	Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4836	}
4837	else
4838	{
4839	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4840	NULL /* pfAddedAndUpdated */);
4841	AssertRCReturn(rc, rc);
4842
4843	/* We need to intercept reads too, see @bugref{7386#c16}. */
4844	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4845	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4846	Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4847	pVCpu->hm.s.vmx.cMsrs));
4848	}
4849	}
4850	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4851	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4852	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4853	}
4854
4855	return VINF_SUCCESS;
4856	}
4857
4858
4859	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4860	/**
4861	* Check if guest state allows safe use of 32-bit switcher again.
4862	*
4863	* Segment bases and protected mode structures must be 32-bit addressable
4864	* because the 32-bit switcher will ignore high dword when writing these VMCS
4865	* fields. See @bugref{8432} for details.
4866	*
4867	* @returns true if safe, false if must continue to use the 64-bit switcher.
4868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4869	* out-of-sync. Make sure to update the required fields
4870	* before using them.
4871	*
4872	* @remarks No-long-jump zone!!!
4873	*/
4874	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4875	{
4876	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4877	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4878	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4879	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4880	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4881	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4882	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4883	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4884	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4885	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4886
4887	/* All good, bases are 32-bit. */
4888	return true;
4889	}
4890	#endif
4891
4892
4893	/**
4894	* Selects up the appropriate function to run guest code.
4895	*
4896	* @returns VBox status code.
4897	* @param pVCpu The cross context virtual CPU structure.
4898	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4899	* out-of-sync. Make sure to update the required fields
4900	* before using them.
4901	*
4902	* @remarks No-long-jump zone!!!
4903	*/
4904	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4905	{
4906	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4907	{
4908	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4909	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4910	#endif
4911	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4912	#if HC_ARCH_BITS == 32
4913	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4914	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4915	{
4916	#ifdef VBOX_STRICT
4917	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4918	{
4919	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4920	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4921	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4922	AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4923	\| HM_CHANGED_VMX_ENTRY_CTLS
4924	\| HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4925	}
4926	#endif
4927	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4928
4929	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4930	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4931	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4932	Log4Func(("Selected 64-bit switcher\n"));
4933	}
4934	#else
4935	/* 64-bit host. */
4936	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4937	#endif
4938	}
4939	else
4940	{
4941	/* Guest is not in long mode, use the 32-bit handler. */
4942	#if HC_ARCH_BITS == 32
4943	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4944	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4945	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4946	{
4947	# ifdef VBOX_STRICT
4948	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4949	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4950	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4951	AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4952	\| HM_CHANGED_VMX_ENTRY_CTLS
4953	\| HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4954	# endif
4955	}
4956	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4957	/*
4958	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4959	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4960	* switcher flag because now we know the guest is in a sane state where it's safe
4961	* to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4962	* the much faster 32-bit switcher again.
4963	*/
4964	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4965	{
4966	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4967	Log4Func(("Selected 32-bit switcher\n"));
4968	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4969	}
4970	else
4971	{
4972	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4973	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4974	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4975	{
4976	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4977	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4978	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4979	\| HM_CHANGED_VMX_ENTRY_CTLS
4980	\| HM_CHANGED_VMX_EXIT_CTLS
4981	\| HM_CHANGED_HOST_CONTEXT);
4982	Log4Func(("Selected 32-bit switcher (safe)\n"));
4983	}
4984	}
4985	# else
4986	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4987	# endif
4988	#else
4989	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4990	#endif
4991	}
4992	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4993	return VINF_SUCCESS;
4994	}
4995
4996
4997	/**
4998	* Wrapper for running the guest code in VT-x.
4999	*
5000	* @returns VBox status code, no informational status codes.
5001	* @param pVCpu The cross context virtual CPU structure.
5002	* @param pCtx Pointer to the guest-CPU context.
5003	*
5004	* @remarks No-long-jump zone!!!
5005	*/
5006	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
5007	{
5008	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
5009	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
5010
5011	/*
5012	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
5013	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
5014	* callee-saved and thus the need for this XMM wrapper.
5015	*
5016	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
5017	*/
5018	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5019	/** @todo Add stats for resume vs launch. */
5020	PVM pVM = pVCpu->CTX_SUFF(pVM);
5021	#ifdef VBOX_WITH_KERNEL_USING_XMM
5022	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5023	#else
5024	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5025	#endif
5026	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5027	return rc;
5028	}
5029
5030
5031	/**
5032	* Reports world-switch error and dumps some useful debug info.
5033	*
5034	* @param pVCpu The cross context virtual CPU structure.
5035	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5036	* @param pCtx Pointer to the guest-CPU context.
5037	* @param pVmxTransient Pointer to the VMX transient structure (only
5038	* exitReason updated).
5039	*/
5040	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5041	{
5042	Assert(pVCpu);
5043	Assert(pCtx);
5044	Assert(pVmxTransient);
5045	HMVMX_ASSERT_PREEMPT_SAFE();
5046
5047	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
5048	switch (rcVMRun)
5049	{
5050	case VERR_VMX_INVALID_VMXON_PTR:
5051	AssertFailed();
5052	break;
5053	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5054	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5055	{
5056	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5057	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5058	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5059	AssertRC(rc);
5060
5061	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5062	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5063	Cannot do it here as we may have been long preempted. */
5064
5065	#ifdef VBOX_STRICT
5066	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5067	pVmxTransient->uExitReason));
5068	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5069	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5070	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5071	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5072	else
5073	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5074	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5075	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5076
5077	/* VMX control bits. */
5078	uint32_t u32Val;
5079	uint64_t u64Val;
5080	RTHCUINTREG uHCReg;
5081	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5082	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5084	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5085	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5086	{
5087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5088	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5089	}
5090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5091	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5093	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5095	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5097	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5098	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5099	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5101	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5103	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5105	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5107	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5109	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5111	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5113	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5115	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5116	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5117	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5118	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5119	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5120	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5121	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5122	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5123	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5124	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5125	{
5126	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5127	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5128	}
5129
5130	/* Guest bits. */
5131	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5132	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5133	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5134	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5135	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5136	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5137	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5138	{
5139	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5140	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5141	}
5142
5143	/* Host bits. */
5144	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5145	Log4(("Host CR0 %#RHr\n", uHCReg));
5146	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5147	Log4(("Host CR3 %#RHr\n", uHCReg));
5148	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5149	Log4(("Host CR4 %#RHr\n", uHCReg));
5150
5151	RTGDTR HostGdtr;
5152	PCX86DESCHC pDesc;
5153	ASMGetGDTR(&HostGdtr);
5154	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5155	Log4(("Host CS %#08x\n", u32Val));
5156	if (u32Val < HostGdtr.cbGdt)
5157	{
5158	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5159	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5160	}
5161
5162	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5163	Log4(("Host DS %#08x\n", u32Val));
5164	if (u32Val < HostGdtr.cbGdt)
5165	{
5166	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5167	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5168	}
5169
5170	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5171	Log4(("Host ES %#08x\n", u32Val));
5172	if (u32Val < HostGdtr.cbGdt)
5173	{
5174	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5175	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5176	}
5177
5178	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5179	Log4(("Host FS %#08x\n", u32Val));
5180	if (u32Val < HostGdtr.cbGdt)
5181	{
5182	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5183	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5184	}
5185
5186	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5187	Log4(("Host GS %#08x\n", u32Val));
5188	if (u32Val < HostGdtr.cbGdt)
5189	{
5190	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5191	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5192	}
5193
5194	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5195	Log4(("Host SS %#08x\n", u32Val));
5196	if (u32Val < HostGdtr.cbGdt)
5197	{
5198	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5199	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5200	}
5201
5202	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5203	Log4(("Host TR %#08x\n", u32Val));
5204	if (u32Val < HostGdtr.cbGdt)
5205	{
5206	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5207	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5208	}
5209
5210	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5211	Log4(("Host TR Base %#RHv\n", uHCReg));
5212	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5213	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5214	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5215	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5216	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5217	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5218	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5219	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5220	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5221	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5222	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5223	Log4(("Host RSP %#RHv\n", uHCReg));
5224	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5225	Log4(("Host RIP %#RHv\n", uHCReg));
5226	# if HC_ARCH_BITS == 64
5227	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5228	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5229	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5230	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5231	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5232	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5233	# endif
5234	#endif /* VBOX_STRICT */
5235	break;
5236	}
5237
5238	default:
5239	/* Impossible */
5240	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5241	break;
5242	}
5243	NOREF(pCtx);
5244	}
5245
5246
5247	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5248	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5249	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5250	#endif
5251	#ifdef VBOX_STRICT
5252	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5253	{
5254	switch (idxField)
5255	{
5256	case VMX_VMCS_GUEST_RIP:
5257	case VMX_VMCS_GUEST_RSP:
5258	case VMX_VMCS_GUEST_SYSENTER_EIP:
5259	case VMX_VMCS_GUEST_SYSENTER_ESP:
5260	case VMX_VMCS_GUEST_GDTR_BASE:
5261	case VMX_VMCS_GUEST_IDTR_BASE:
5262	case VMX_VMCS_GUEST_CS_BASE:
5263	case VMX_VMCS_GUEST_DS_BASE:
5264	case VMX_VMCS_GUEST_ES_BASE:
5265	case VMX_VMCS_GUEST_FS_BASE:
5266	case VMX_VMCS_GUEST_GS_BASE:
5267	case VMX_VMCS_GUEST_SS_BASE:
5268	case VMX_VMCS_GUEST_LDTR_BASE:
5269	case VMX_VMCS_GUEST_TR_BASE:
5270	case VMX_VMCS_GUEST_CR3:
5271	return true;
5272	}
5273	return false;
5274	}
5275
5276	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5277	{
5278	switch (idxField)
5279	{
5280	/* Read-only fields. */
5281	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5282	return true;
5283	}
5284	/* Remaining readable fields should also be writable. */
5285	return hmR0VmxIsValidWriteField(idxField);
5286	}
5287	#endif /* VBOX_STRICT */
5288
5289
5290	/**
5291	* Executes the specified handler in 64-bit mode.
5292	*
5293	* @returns VBox status code (no informational status codes).
5294	* @param pVCpu The cross context virtual CPU structure.
5295	* @param enmOp The operation to perform.
5296	* @param cParams Number of parameters.
5297	* @param paParam Array of 32-bit parameters.
5298	*/
5299	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5300	{
5301	PVM pVM = pVCpu->CTX_SUFF(pVM);
5302	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5303	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5304	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5305	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5306
5307	#ifdef VBOX_STRICT
5308	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5309	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5310
5311	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5312	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5313	#endif
5314
5315	/* Disable interrupts. */
5316	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5317
5318	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5319	RTCPUID idHostCpu = RTMpCpuId();
5320	CPUMR0SetLApic(pVCpu, idHostCpu);
5321	#endif
5322
5323	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5324	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5325
5326	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5327	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5328	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5329
5330	/* Leave VMX Root Mode. */
5331	VMXDisable();
5332
5333	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5334
5335	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5336	CPUMSetHyperEIP(pVCpu, enmOp);
5337	for (int i = (int)cParams - 1; i >= 0; i--)
5338	CPUMPushHyper(pVCpu, paParam[i]);
5339
5340	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5341
5342	/* Call the switcher. */
5343	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5344	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5345
5346	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5347	/* Make sure the VMX instructions don't cause #UD faults. */
5348	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5349
5350	/* Re-enter VMX Root Mode */
5351	int rc2 = VMXEnable(HCPhysCpuPage);
5352	if (RT_FAILURE(rc2))
5353	{
5354	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5355	ASMSetFlags(fOldEFlags);
5356	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5357	return rc2;
5358	}
5359
5360	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5361	AssertRC(rc2);
5362	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5363	Assert(!(ASMGetFlags() & X86_EFL_IF));
5364	ASMSetFlags(fOldEFlags);
5365	return rc;
5366	}
5367
5368
5369	/**
5370	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5371	* supporting 64-bit guests.
5372	*
5373	* @returns VBox status code.
5374	* @param fResume Whether to VMLAUNCH or VMRESUME.
5375	* @param pCtx Pointer to the guest-CPU context.
5376	* @param pCache Pointer to the VMCS cache.
5377	* @param pVM The cross context VM structure.
5378	* @param pVCpu The cross context virtual CPU structure.
5379	*/
5380	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5381	{
5382	NOREF(fResume);
5383
5384	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5385	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5386
5387	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5388	pCache->uPos = 1;
5389	pCache->interPD = PGMGetInterPaeCR3(pVM);
5390	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5391	#endif
5392
5393	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5394	pCache->TestIn.HCPhysCpuPage = 0;
5395	pCache->TestIn.HCPhysVmcs = 0;
5396	pCache->TestIn.pCache = 0;
5397	pCache->TestOut.HCPhysVmcs = 0;
5398	pCache->TestOut.pCache = 0;
5399	pCache->TestOut.pCtx = 0;
5400	pCache->TestOut.eflags = 0;
5401	#else
5402	NOREF(pCache);
5403	#endif
5404
5405	uint32_t aParam[10];
5406	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5407	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5408	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5409	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5410	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5411	aParam[5] = 0;
5412	aParam[6] = VM_RC_ADDR(pVM, pVM);
5413	aParam[7] = 0;
5414	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5415	aParam[9] = 0;
5416
5417	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5418	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5419	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5420	#endif
5421	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5422
5423	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5424	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5425	Assert(pCtx->dr[4] == 10);
5426	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5427	#endif
5428
5429	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5430	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5431	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5432	pVCpu->hm.s.vmx.HCPhysVmcs));
5433	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5434	pCache->TestOut.HCPhysVmcs));
5435	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5436	pCache->TestOut.pCache));
5437	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5438	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5439	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5440	pCache->TestOut.pCtx));
5441	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5442	#endif
5443	NOREF(pCtx);
5444	return rc;
5445	}
5446
5447
5448	/**
5449	* Initialize the VMCS-Read cache.
5450	*
5451	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5452	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5453	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5454	* (those that have a 32-bit FULL & HIGH part).
5455	*
5456	* @returns VBox status code.
5457	* @param pVCpu The cross context virtual CPU structure.
5458	*/
5459	static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5460	{
5461	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5462	do { \
5463	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5464	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5465	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5466	++cReadFields; \
5467	} while (0)
5468
5469	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5470	uint32_t cReadFields = 0;
5471
5472	/*
5473	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5474	* and serve to indicate exceptions to the rules.
5475	*/
5476
5477	/* Guest-natural selector base fields. */
5478	#if 0
5479	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5480	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5481	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5482	#endif
5483	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5484	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5485	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5486	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5487	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5488	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5489	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5490	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5491	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5492	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5493	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5494	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5495	#if 0
5496	/* Unused natural width guest-state fields. */
5497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5498	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5499	#endif
5500	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5501	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5502
5503	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5504	these 64-bit fields (using "FULL" and "HIGH" fields). */
5505	#if 0
5506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5511	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5512	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5513	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5515	#endif
5516
5517	/* Natural width guest-state fields. */
5518	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5519	#if 0
5520	/* Currently unused field. */
5521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5522	#endif
5523
5524	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5525	{
5526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5527	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5528	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5529	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5530	}
5531	else
5532	{
5533	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5534	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5535	}
5536
5537	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5538	return VINF_SUCCESS;
5539	}
5540
5541
5542	/**
5543	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5544	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5545	* darwin, running 64-bit guests).
5546	*
5547	* @returns VBox status code.
5548	* @param pVCpu The cross context virtual CPU structure.
5549	* @param idxField The VMCS field encoding.
5550	* @param u64Val 16, 32 or 64-bit value.
5551	*/
5552	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5553	{
5554	int rc;
5555	switch (idxField)
5556	{
5557	/*
5558	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5559	*/
5560	/* 64-bit Control fields. */
5561	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5562	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5563	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5564	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5565	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5566	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5567	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5568	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5569	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5570	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5571	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5572	case VMX_VMCS64_CTRL_EPTP_FULL:
5573	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5574	/* 64-bit Guest-state fields. */
5575	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5576	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5577	case VMX_VMCS64_GUEST_PAT_FULL:
5578	case VMX_VMCS64_GUEST_EFER_FULL:
5579	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5580	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5581	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5582	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5583	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5584	/* 64-bit Host-state fields. */
5585	case VMX_VMCS64_HOST_PAT_FULL:
5586	case VMX_VMCS64_HOST_EFER_FULL:
5587	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5588	{
5589	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5590	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5591	break;
5592	}
5593
5594	/*
5595	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5596	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5597	*/
5598	/* Natural-width Guest-state fields. */
5599	case VMX_VMCS_GUEST_CR3:
5600	case VMX_VMCS_GUEST_ES_BASE:
5601	case VMX_VMCS_GUEST_CS_BASE:
5602	case VMX_VMCS_GUEST_SS_BASE:
5603	case VMX_VMCS_GUEST_DS_BASE:
5604	case VMX_VMCS_GUEST_FS_BASE:
5605	case VMX_VMCS_GUEST_GS_BASE:
5606	case VMX_VMCS_GUEST_LDTR_BASE:
5607	case VMX_VMCS_GUEST_TR_BASE:
5608	case VMX_VMCS_GUEST_GDTR_BASE:
5609	case VMX_VMCS_GUEST_IDTR_BASE:
5610	case VMX_VMCS_GUEST_RSP:
5611	case VMX_VMCS_GUEST_RIP:
5612	case VMX_VMCS_GUEST_SYSENTER_ESP:
5613	case VMX_VMCS_GUEST_SYSENTER_EIP:
5614	{
5615	if (!(RT_HI_U32(u64Val)))
5616	{
5617	/* If this field is 64-bit, VT-x will zero out the top bits. */
5618	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5619	}
5620	else
5621	{
5622	/* Assert that only the 32->64 switcher case should ever come here. */
5623	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5624	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5625	}
5626	break;
5627	}
5628
5629	default:
5630	{
5631	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5632	rc = VERR_INVALID_PARAMETER;
5633	break;
5634	}
5635	}
5636	AssertRCReturn(rc, rc);
5637	return rc;
5638	}
5639
5640
5641	/**
5642	* Queue up a VMWRITE by using the VMCS write cache.
5643	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5644	*
5645	* @param pVCpu The cross context virtual CPU structure.
5646	* @param idxField The VMCS field encoding.
5647	* @param u64Val 16, 32 or 64-bit value.
5648	*/
5649	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5650	{
5651	AssertPtr(pVCpu);
5652	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5653
5654	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5655	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5656
5657	/* Make sure there are no duplicates. */
5658	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5659	{
5660	if (pCache->Write.aField[i] == idxField)
5661	{
5662	pCache->Write.aFieldVal[i] = u64Val;
5663	return VINF_SUCCESS;
5664	}
5665	}
5666
5667	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5668	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5669	pCache->Write.cValidEntries++;
5670	return VINF_SUCCESS;
5671	}
5672	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5673
5674
5675	/**
5676	* Sets up the usage of TSC-offsetting and updates the VMCS.
5677	*
5678	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5679	* VMX preemption timer.
5680	*
5681	* @returns VBox status code.
5682	* @param pVCpu The cross context virtual CPU structure.
5683	*
5684	* @remarks No-long-jump zone!!!
5685	*/
5686	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5687	{
5688	bool fOffsettedTsc;
5689	bool fParavirtTsc;
5690	PVM pVM = pVCpu->CTX_SUFF(pVM);
5691	uint64_t uTscOffset;
5692	if (pVM->hm.s.vmx.fUsePreemptTimer)
5693	{
5694	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5695
5696	/* Make sure the returned values have sane upper and lower boundaries. */
5697	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5698	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5699	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5700	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5701
5702	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5703	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5704	AssertRC(rc);
5705	}
5706	else
5707	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5708
5709	/** @todo later optimize this to be done elsewhere and not before every
5710	* VM-entry. */
5711	if (fParavirtTsc)
5712	{
5713	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5714	information before every VM-entry, hence disable it for performance sake. */
5715	#if 0
5716	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5717	AssertRC(rc);
5718	#endif
5719	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5720	}
5721
5722	uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5723	if ( fOffsettedTsc
5724	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5725	{
5726	if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5727	{
5728	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5729	AssertRC(rc);
5730	pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5731	}
5732
5733	if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5734	{
5735	uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5736	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5737	AssertRC(rc);
5738	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5739	}
5740	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5741	}
5742	else
5743	{
5744	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5745	if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5746	{
5747	uProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5748	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5749	AssertRC(rc);
5750	pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5751	}
5752	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5753	}
5754	}
5755
5756
5757	/**
5758	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5759	* VM-exit interruption info type.
5760	*
5761	* @returns The IEM exception flags.
5762	* @param uVector The event vector.
5763	* @param uVmxVectorType The VMX event type.
5764	*
5765	* @remarks This function currently only constructs flags required for
5766	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5767	* and CR2 aspects of an exception are not included).
5768	*/
5769	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5770	{
5771	uint32_t fIemXcptFlags;
5772	switch (uVmxVectorType)
5773	{
5774	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5775	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5776	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5777	break;
5778
5779	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5780	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5781	break;
5782
5783	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5784	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5785	break;
5786
5787	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5788	{
5789	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5790	if (uVector == X86_XCPT_BP)
5791	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5792	else if (uVector == X86_XCPT_OF)
5793	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5794	else
5795	{
5796	fIemXcptFlags = 0;
5797	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5798	}
5799	break;
5800	}
5801
5802	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5803	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5804	break;
5805
5806	default:
5807	fIemXcptFlags = 0;
5808	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5809	break;
5810	}
5811	return fIemXcptFlags;
5812	}
5813
5814
5815	/**
5816	* Sets an event as a pending event to be injected into the guest.
5817	*
5818	* @param pVCpu The cross context virtual CPU structure.
5819	* @param u32IntInfo The VM-entry interruption-information field.
5820	* @param cbInstr The VM-entry instruction length in bytes (for software
5821	* interrupts, exceptions and privileged software
5822	* exceptions).
5823	* @param u32ErrCode The VM-entry exception error code.
5824	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5825	* page-fault.
5826	*
5827	* @remarks Statistics counter assumes this is a guest event being injected or
5828	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5829	* always incremented.
5830	*/
5831	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5832	RTGCUINTPTR GCPtrFaultAddress)
5833	{
5834	Assert(!pVCpu->hm.s.Event.fPending);
5835	pVCpu->hm.s.Event.fPending = true;
5836	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5837	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5838	pVCpu->hm.s.Event.cbInstr = cbInstr;
5839	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5840	}
5841
5842
5843	/**
5844	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5845	*
5846	* @param pVCpu The cross context virtual CPU structure.
5847	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5848	* out-of-sync. Make sure to update the required fields
5849	* before using them.
5850	*/
5851	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5852	{
5853	NOREF(pMixedCtx);
5854	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5855	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5856	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5857	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5858	}
5859
5860
5861	/**
5862	* Handle a condition that occurred while delivering an event through the guest
5863	* IDT.
5864	*
5865	* @returns Strict VBox status code (i.e. informational status codes too).
5866	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5867	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5868	* to continue execution of the guest which will delivery the \#DF.
5869	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5870	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5871	*
5872	* @param pVCpu The cross context virtual CPU structure.
5873	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5874	* out-of-sync. Make sure to update the required fields
5875	* before using them.
5876	* @param pVmxTransient Pointer to the VMX transient structure.
5877	*
5878	* @remarks No-long-jump zone!!!
5879	*/
5880	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5881	{
5882	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5883
5884	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5885	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5886	AssertRCReturn(rc2, rc2);
5887
5888	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5889	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5890	{
5891	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5892	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5893
5894	/*
5895	* If the event was a software interrupt (generated with INT n) or a software exception
5896	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5897	* can handle the VM-exit and continue guest execution which will re-execute the
5898	* instruction rather than re-injecting the exception, as that can cause premature
5899	* trips to ring-3 before injection and involve TRPM which currently has no way of
5900	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5901	* the problem).
5902	*/
5903	IEMXCPTRAISE enmRaise;
5904	IEMXCPTRAISEINFO fRaiseInfo;
5905	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5906	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5907	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5908	{
5909	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5910	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5911	}
5912	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5913	{
5914	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5915	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5916	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5917	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5918	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5919	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5920	uExitVectorType), VERR_VMX_IPE_5);
5921
5922	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5923
5924	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5925	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5926	{
5927	pVmxTransient->fVectoringPF = true;
5928	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5929	}
5930	}
5931	else
5932	{
5933	/*
5934	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5935	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5936	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5937	*/
5938	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5939	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5940	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5941	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5942	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5943	}
5944
5945	/*
5946	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5947	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5948	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5949	* subsequent VM-entry would fail.
5950	*
5951	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5952	*/
5953	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5954	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5955	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5956	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5957	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5958	{
5959	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5960	}
5961
5962	switch (enmRaise)
5963	{
5964	case IEMXCPTRAISE_CURRENT_XCPT:
5965	{
5966	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5967	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5968	Assert(rcStrict == VINF_SUCCESS);
5969	break;
5970	}
5971
5972	case IEMXCPTRAISE_PREV_EVENT:
5973	{
5974	uint32_t u32ErrCode;
5975	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5976	{
5977	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5978	AssertRCReturn(rc2, rc2);
5979	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5980	}
5981	else
5982	u32ErrCode = 0;
5983
5984	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5985	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5986	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5987	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5988
5989	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5990	pVCpu->hm.s.Event.u32ErrCode));
5991	Assert(rcStrict == VINF_SUCCESS);
5992	break;
5993	}
5994
5995	case IEMXCPTRAISE_REEXEC_INSTR:
5996	Assert(rcStrict == VINF_SUCCESS);
5997	break;
5998
5999	case IEMXCPTRAISE_DOUBLE_FAULT:
6000	{
6001	/*
6002	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6003	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6004	*/
6005	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6006	{
6007	pVmxTransient->fVectoringDoublePF = true;
6008	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6009	pMixedCtx->cr2));
6010	rcStrict = VINF_SUCCESS;
6011	}
6012	else
6013	{
6014	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6015	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6016	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6017	uIdtVector, uExitVector));
6018	rcStrict = VINF_HM_DOUBLE_FAULT;
6019	}
6020	break;
6021	}
6022
6023	case IEMXCPTRAISE_TRIPLE_FAULT:
6024	{
6025	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6026	rcStrict = VINF_EM_RESET;
6027	break;
6028	}
6029
6030	case IEMXCPTRAISE_CPU_HANG:
6031	{
6032	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6033	rcStrict = VERR_EM_GUEST_CPU_HANG;
6034	break;
6035	}
6036
6037	default:
6038	{
6039	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6040	rcStrict = VERR_VMX_IPE_2;
6041	break;
6042	}
6043	}
6044	}
6045	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6046	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6047	&& uExitVector != X86_XCPT_DF
6048	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6049	{
6050	/*
6051	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6052	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6053	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6054	*/
6055	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6056	{
6057	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6058	VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6059	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6060	}
6061	}
6062
6063	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6064	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6065	return rcStrict;
6066	}
6067
6068
6069	/**
6070	* Imports a guest segment register from the current VMCS into
6071	* the guest-CPU context.
6072	*
6073	* @returns VBox status code.
6074	* @param pVCpu The cross context virtual CPU structure.
6075	* @param idxSel Index of the selector in the VMCS.
6076	* @param idxLimit Index of the segment limit in the VMCS.
6077	* @param idxBase Index of the segment base in the VMCS.
6078	* @param idxAccess Index of the access rights of the segment in the VMCS.
6079	* @param pSelReg Pointer to the segment selector.
6080	*
6081	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
6082	* do not log!
6083	*
6084	* @remarks Never call this function directly!!! Use the
6085	* HMVMX_IMPORT_SREG() macro as that takes care
6086	* of whether to read from the VMCS cache or not.
6087	*/
6088	static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6089	PCPUMSELREG pSelReg)
6090	{
6091	NOREF(pVCpu);
6092
6093	uint32_t u32Sel;
6094	uint32_t u32Limit;
6095	uint32_t u32Attr;
6096	uint64_t u64Base;
6097	int rc = VMXReadVmcs32(idxSel, &u32Sel);
6098	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
6099	rc \|= VMXReadVmcs32(idxAccess, &u32Attr);
6100	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6101	AssertRCReturn(rc, rc);
6102
6103	pSelReg->Sel = (uint16_t)u32Sel;
6104	pSelReg->ValidSel = (uint16_t)u32Sel;
6105	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6106	pSelReg->u32Limit = u32Limit;
6107	pSelReg->u64Base = u64Base;
6108	pSelReg->Attr.u = u32Attr;
6109
6110	/*
6111	* If VT-x marks the segment as unusable, most other bits remain undefined:
6112	* - For CS the L, D and G bits have meaning.
6113	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6114	* - For the remaining data segments no bits are defined.
6115	*
6116	* The present bit and the unusable bit has been observed to be set at the
6117	* same time (the selector was supposed to be invalid as we started executing
6118	* a V8086 interrupt in ring-0).
6119	*
6120	* What should be important for the rest of the VBox code, is that the P bit is
6121	* cleared. Some of the other VBox code recognizes the unusable bit, but
6122	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6123	* safe side here, we'll strip off P and other bits we don't care about. If
6124	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6125	*
6126	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6127	*/
6128	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6129	{
6130	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6131
6132	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6133	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6134	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6135	#ifdef VBOX_STRICT
6136	VMMRZCallRing3Disable(pVCpu);
6137	Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6138	# ifdef DEBUG_bird
6139	AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6140	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6141	idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6142	# endif
6143	VMMRZCallRing3Enable(pVCpu);
6144	#endif
6145	}
6146	return VINF_SUCCESS;
6147	}
6148
6149
6150	/**
6151	* Imports the guest RIP from the VMCS back into the guest-CPU context.
6152	*
6153	* @returns VBox status code.
6154	* @param pVCpu The cross context virtual CPU structure.
6155	*
6156	* @remarks Called with interrupts and/or preemption disabled, should not assert!
6157	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6158	* instead!!!
6159	*/
6160	DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6161	{
6162	uint64_t u64Val;
6163	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6164	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6165	{
6166	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6167	if (RT_SUCCESS(rc))
6168	{
6169	pCtx->rip = u64Val;
6170	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6171	}
6172	return rc;
6173	}
6174	return VINF_SUCCESS;
6175	}
6176
6177
6178	/**
6179	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6180	*
6181	* @returns VBox status code.
6182	* @param pVCpu The cross context virtual CPU structure.
6183	*
6184	* @remarks Called with interrupts and/or preemption disabled, should not assert!
6185	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6186	* instead!!!
6187	*/
6188	DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6189	{
6190	uint32_t u32Val;
6191	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6192	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6193	{
6194	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6195	if (RT_SUCCESS(rc))
6196	{
6197	pCtx->eflags.u32 = u32Val;
6198
6199	/* Restore eflags for real-on-v86-mode hack. */
6200	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6201	{
6202	pCtx->eflags.Bits.u1VM = 0;
6203	pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6204	}
6205	}
6206	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6207	return rc;
6208	}
6209	return VINF_SUCCESS;
6210	}
6211
6212
6213	/**
6214	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6215	* context.
6216	*
6217	* @returns VBox status code.
6218	* @param pVCpu The cross context virtual CPU structure.
6219	*
6220	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
6221	* do not log!
6222	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6223	* instead!!!
6224	*/
6225	DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6226	{
6227	uint32_t u32Val;
6228	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6229	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6230	if (RT_SUCCESS(rc))
6231	{
6232	/*
6233	* We additionally have a requirement to import RIP, RFLAGS depending on whether we
6234	* might need them in hmR0VmxEvaluatePendingEvent().
6235	*/
6236	if (!u32Val)
6237	{
6238	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6239	{
6240	rc = hmR0VmxImportGuestRip(pVCpu);
6241	rc \|= hmR0VmxImportGuestRFlags(pVCpu);
6242	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6243	}
6244
6245	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6246	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6247	}
6248	else
6249	{
6250	rc = hmR0VmxImportGuestRip(pVCpu);
6251	rc \|= hmR0VmxImportGuestRFlags(pVCpu);
6252
6253	if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6254	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6255	{
6256	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6257	}
6258	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6259	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6260
6261	if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6262	{
6263	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6264	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6265	}
6266	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6267	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6268	}
6269	}
6270	return rc;
6271	}
6272
6273
6274	/**
6275	* Worker for VMXR0ImportStateOnDemand.
6276	*
6277	* @returns VBox status code.
6278	* @param pVCpu The cross context virtual CPU structure.
6279	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6280	*/
6281	static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6282	{
6283	#define VMXLOCAL_BREAK_RC(a_rc) \
6284	if (RT_FAILURE(a_rc)) \
6285	break
6286
6287	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6288
6289	int rc = VINF_SUCCESS;
6290	PVM pVM = pVCpu->CTX_SUFF(pVM);
6291	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6292	uint64_t u64Val;
6293	uint32_t u32Val;
6294
6295	Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6296
6297	/*
6298	* We disable interrupts to make the updating of the state and in particular
6299	* the fExtrn modification atomic wrt to preemption hooks.
6300	*/
6301	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6302
6303	fWhat &= pCtx->fExtrn;
6304	if (fWhat)
6305	{
6306	do
6307	{
6308	if (fWhat & CPUMCTX_EXTRN_RIP)
6309	{
6310	rc = hmR0VmxImportGuestRip(pVCpu);
6311	VMXLOCAL_BREAK_RC(rc);
6312	}
6313
6314	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6315	{
6316	rc = hmR0VmxImportGuestRFlags(pVCpu);
6317	VMXLOCAL_BREAK_RC(rc);
6318	}
6319
6320	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6321	{
6322	rc = hmR0VmxImportGuestIntrState(pVCpu);
6323	VMXLOCAL_BREAK_RC(rc);
6324	}
6325
6326	if (fWhat & CPUMCTX_EXTRN_RSP)
6327	{
6328	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6329	VMXLOCAL_BREAK_RC(rc);
6330	pCtx->rsp = u64Val;
6331	}
6332
6333	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6334	{
6335	if (fWhat & CPUMCTX_EXTRN_CS)
6336	{
6337	rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6338	VMXLOCAL_BREAK_RC(rc);
6339	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6340	pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6341	}
6342	if (fWhat & CPUMCTX_EXTRN_SS)
6343	{
6344	rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6345	VMXLOCAL_BREAK_RC(rc);
6346	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6347	pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6348	}
6349	if (fWhat & CPUMCTX_EXTRN_DS)
6350	{
6351	rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6352	VMXLOCAL_BREAK_RC(rc);
6353	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6354	pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6355	}
6356	if (fWhat & CPUMCTX_EXTRN_ES)
6357	{
6358	rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6359	VMXLOCAL_BREAK_RC(rc);
6360	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6361	pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6362	}
6363	if (fWhat & CPUMCTX_EXTRN_FS)
6364	{
6365	rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6366	VMXLOCAL_BREAK_RC(rc);
6367	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6368	pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6369	}
6370	if (fWhat & CPUMCTX_EXTRN_GS)
6371	{
6372	rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6373	VMXLOCAL_BREAK_RC(rc);
6374	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6375	pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6376	}
6377	}
6378
6379	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6380	{
6381	if (fWhat & CPUMCTX_EXTRN_LDTR)
6382	{
6383	rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6384	VMXLOCAL_BREAK_RC(rc);
6385	}
6386
6387	if (fWhat & CPUMCTX_EXTRN_GDTR)
6388	{
6389	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6390	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6391	VMXLOCAL_BREAK_RC(rc);
6392	pCtx->gdtr.pGdt = u64Val;
6393	pCtx->gdtr.cbGdt = u32Val;
6394	}
6395
6396	/* Guest IDTR. */
6397	if (fWhat & CPUMCTX_EXTRN_IDTR)
6398	{
6399	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6400	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6401	VMXLOCAL_BREAK_RC(rc);
6402	pCtx->idtr.pIdt = u64Val;
6403	pCtx->idtr.cbIdt = u32Val;
6404	}
6405
6406	/* Guest TR. */
6407	if (fWhat & CPUMCTX_EXTRN_TR)
6408	{
6409	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6410	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6411	{
6412	rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6413	VMXLOCAL_BREAK_RC(rc);
6414	}
6415	}
6416	}
6417
6418	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6419	{
6420	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6421	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6422	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6423	pCtx->SysEnter.cs = u32Val;
6424	VMXLOCAL_BREAK_RC(rc);
6425	}
6426
6427	#if HC_ARCH_BITS == 64
6428	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6429	{
6430	if ( pVM->hm.s.fAllow64BitGuests
6431	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6432	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6433	}
6434
6435	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6436	{
6437	if ( pVM->hm.s.fAllow64BitGuests
6438	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6439	{
6440	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6441	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6442	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6443	}
6444	}
6445	#endif
6446
6447	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
6448	#if HC_ARCH_BITS == 32
6449	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
6450	#endif
6451	)
6452	{
6453	PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6454	uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6455	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6456	{
6457	switch (pMsr->u32Msr)
6458	{
6459	#if HC_ARCH_BITS == 32
6460	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6461	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6462	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6463	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6464	#endif
6465	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6466	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6467	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6468	default:
6469	{
6470	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6471	ASMSetFlags(fEFlags);
6472	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6473	cMsrs));
6474	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6475	}
6476	}
6477	}
6478	}
6479
6480	if (fWhat & CPUMCTX_EXTRN_DR7)
6481	{
6482	if (!pVCpu->hm.s.fUsingHyperDR7)
6483	{
6484	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6485	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6486	VMXLOCAL_BREAK_RC(rc);
6487	pCtx->dr[7] = u32Val;
6488	}
6489	}
6490
6491	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6492	{
6493	uint32_t u32Shadow;
6494	if (fWhat & CPUMCTX_EXTRN_CR0)
6495	{
6496	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6497	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6498	VMXLOCAL_BREAK_RC(rc);
6499	u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6500	\| (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6501	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6502	CPUMSetGuestCR0(pVCpu, u32Val);
6503	VMMRZCallRing3Enable(pVCpu);
6504	}
6505
6506	if (fWhat & CPUMCTX_EXTRN_CR4)
6507	{
6508	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6509	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6510	VMXLOCAL_BREAK_RC(rc);
6511	u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6512	\| (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6513	CPUMSetGuestCR4(pVCpu, u32Val);
6514	}
6515
6516	if (fWhat & CPUMCTX_EXTRN_CR3)
6517	{
6518	/* CR0.PG bit changes are always intercepted, so it's up to date. */
6519	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6520	\|\| ( pVM->hm.s.fNestedPaging
6521	&& CPUMIsGuestPagingEnabledEx(pCtx)))
6522	{
6523	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6524	if (pCtx->cr3 != u64Val)
6525	{
6526	CPUMSetGuestCR3(pVCpu, u64Val);
6527	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6528	}
6529
6530	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6531	Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6532	if (CPUMIsGuestInPAEModeEx(pCtx))
6533	{
6534	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6535	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6536	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6537	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6538	VMXLOCAL_BREAK_RC(rc);
6539	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6540	}
6541	}
6542	}
6543	}
6544	} while (0);
6545
6546	if (RT_SUCCESS(rc))
6547	{
6548	/* Update fExtrn. */
6549	pCtx->fExtrn &= ~fWhat;
6550
6551	/* If everything has been imported, clear the HM keeper bit. */
6552	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6553	{
6554	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6555	Assert(!pCtx->fExtrn);
6556	}
6557	}
6558	}
6559	else
6560	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6561
6562	ASMSetFlags(fEFlags);
6563
6564	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6565
6566	/*
6567	* Honor any pending CR3 updates.
6568	*
6569	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6570	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6571	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6572	*
6573	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6574	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6575	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6576	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6577	*
6578	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6579	*/
6580	if (VMMRZCallRing3IsEnabled(pVCpu))
6581	{
6582	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6583	{
6584	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6585	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6586	}
6587
6588	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6589	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6590
6591	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6592	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6593	}
6594
6595	return VINF_SUCCESS;
6596	#undef VMXLOCAL_BREAK_RC
6597	}
6598
6599
6600	/**
6601	* Saves the guest state from the VMCS into the guest-CPU context.
6602	*
6603	* @returns VBox status code.
6604	* @param pVCpu The cross context virtual CPU structure.
6605	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6606	*/
6607	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6608	{
6609	return hmR0VmxImportGuestState(pVCpu, fWhat);
6610	}
6611
6612
6613	/**
6614	* Check per-VM and per-VCPU force flag actions that require us to go back to
6615	* ring-3 for one reason or another.
6616	*
6617	* @returns Strict VBox status code (i.e. informational status codes too)
6618	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6619	* ring-3.
6620	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6621	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6622	* interrupts)
6623	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6624	* all EMTs to be in ring-3.
6625	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6626	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6627	* to the EM loop.
6628	*
6629	* @param pVCpu The cross context virtual CPU structure.
6630	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6631	* out-of-sync. Make sure to update the required fields
6632	* before using them.
6633	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
6634	*/
6635	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6636	{
6637	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6638
6639	/*
6640	* Anything pending? Should be more likely than not if we're doing a good job.
6641	*/
6642	PVM pVM = pVCpu->CTX_SUFF(pVM);
6643	if ( !fStepping
6644	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6645	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6646	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6647	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6648	return VINF_SUCCESS;
6649
6650	/* Pending PGM C3 sync. */
6651	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6652	{
6653	Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
6654	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6655	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6656	if (rcStrict2 != VINF_SUCCESS)
6657	{
6658	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6659	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6660	return rcStrict2;
6661	}
6662	}
6663
6664	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6665	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6666	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6667	{
6668	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6669	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6670	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6671	return rc2;
6672	}
6673
6674	/* Pending VM request packets, such as hardware interrupts. */
6675	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6676	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6677	{
6678	Log4Func(("Pending VM request forcing us back to ring-3\n"));
6679	return VINF_EM_PENDING_REQUEST;
6680	}
6681
6682	/* Pending PGM pool flushes. */
6683	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6684	{
6685	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6686	return VINF_PGM_POOL_FLUSH_PENDING;
6687	}
6688
6689	/* Pending DMA requests. */
6690	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6691	{
6692	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6693	return VINF_EM_RAW_TO_R3;
6694	}
6695
6696	return VINF_SUCCESS;
6697	}
6698
6699
6700	/**
6701	* Converts any TRPM trap into a pending HM event. This is typically used when
6702	* entering from ring-3 (not longjmp returns).
6703	*
6704	* @param pVCpu The cross context virtual CPU structure.
6705	*/
6706	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6707	{
6708	Assert(TRPMHasTrap(pVCpu));
6709	Assert(!pVCpu->hm.s.Event.fPending);
6710
6711	uint8_t uVector;
6712	TRPMEVENT enmTrpmEvent;
6713	RTGCUINT uErrCode;
6714	RTGCUINTPTR GCPtrFaultAddress;
6715	uint8_t cbInstr;
6716
6717	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6718	AssertRC(rc);
6719
6720	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6721	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6722	if (enmTrpmEvent == TRPM_TRAP)
6723	{
6724	switch (uVector)
6725	{
6726	case X86_XCPT_NMI:
6727	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6728	break;
6729
6730	case X86_XCPT_BP:
6731	case X86_XCPT_OF:
6732	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6733	break;
6734
6735	case X86_XCPT_PF:
6736	case X86_XCPT_DF:
6737	case X86_XCPT_TS:
6738	case X86_XCPT_NP:
6739	case X86_XCPT_SS:
6740	case X86_XCPT_GP:
6741	case X86_XCPT_AC:
6742	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6743	RT_FALL_THRU();
6744	default:
6745	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6746	break;
6747	}
6748	}
6749	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6750	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6751	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6752	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6753	else
6754	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6755
6756	rc = TRPMResetTrap(pVCpu);
6757	AssertRC(rc);
6758	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6759	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6760
6761	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6762	}
6763
6764
6765	/**
6766	* Converts the pending HM event into a TRPM trap.
6767	*
6768	* @param pVCpu The cross context virtual CPU structure.
6769	*/
6770	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6771	{
6772	Assert(pVCpu->hm.s.Event.fPending);
6773
6774	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6775	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6776	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6777	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6778
6779	/* If a trap was already pending, we did something wrong! */
6780	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6781
6782	TRPMEVENT enmTrapType;
6783	switch (uVectorType)
6784	{
6785	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6786	enmTrapType = TRPM_HARDWARE_INT;
6787	break;
6788
6789	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6790	enmTrapType = TRPM_SOFTWARE_INT;
6791	break;
6792
6793	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6794	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6795	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6796	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6797	enmTrapType = TRPM_TRAP;
6798	break;
6799
6800	default:
6801	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6802	enmTrapType = TRPM_32BIT_HACK;
6803	break;
6804	}
6805
6806	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6807
6808	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6809	AssertRC(rc);
6810
6811	if (fErrorCodeValid)
6812	TRPMSetErrorCode(pVCpu, uErrorCode);
6813
6814	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6815	&& uVector == X86_XCPT_PF)
6816	{
6817	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6818	}
6819	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6820	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6821	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6822	{
6823	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6824	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6825	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6826	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6827	}
6828
6829	/* Clear any pending events from the VMCS. */
6830	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6831	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6832
6833	/* We're now done converting the pending event. */
6834	pVCpu->hm.s.Event.fPending = false;
6835	}
6836
6837
6838	/**
6839	* Does the necessary state syncing before returning to ring-3 for any reason
6840	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6841	*
6842	* @returns VBox status code.
6843	* @param pVCpu The cross context virtual CPU structure.
6844	* @param fImportState Whether to import the guest state from the VMCS back
6845	* to the guest-CPU context.
6846	*
6847	* @remarks No-long-jmp zone!!!
6848	*/
6849	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6850	{
6851	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6852	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6853
6854	RTCPUID idCpu = RTMpCpuId();
6855	Log4Func(("HostCpuId=%u\n", idCpu));
6856
6857	/*
6858	* !!! IMPORTANT !!!
6859	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6860	*/
6861
6862	/* Save the guest state if necessary. */
6863	if (fImportState)
6864	{
6865	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6866	AssertRCReturn(rc, rc);
6867	}
6868
6869	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6870	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6871	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6872
6873	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6874	#ifdef VBOX_STRICT
6875	if (CPUMIsHyperDebugStateActive(pVCpu))
6876	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6877	#endif
6878	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6879	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6880	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6881
6882	#if HC_ARCH_BITS == 64
6883	/* Restore host-state bits that VT-x only restores partially. */
6884	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6885	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6886	{
6887	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6888	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6889	}
6890	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6891	#endif
6892
6893	/* Restore the lazy host MSRs as we're leaving VT-x context. */
6894	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6895	{
6896	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6897	if (!fImportState)
6898	{
6899	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6900	\| CPUMCTX_EXTRN_SYSCALL_MSRS);
6901	AssertRCReturn(rc, rc);
6902	}
6903	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6904	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6905	}
6906	else
6907	pVCpu->hm.s.vmx.fLazyMsrs = 0;
6908
6909	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6910	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6911
6912	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6913	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6914	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6915	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6916	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6917	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6918	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6919	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6920	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6921
6922	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6923
6924	/** @todo This partially defeats the purpose of having preemption hooks.
6925	* The problem is, deregistering the hooks should be moved to a place that
6926	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6927	* context.
6928	*/
6929	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6930	{
6931	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6932	AssertRCReturn(rc, rc);
6933
6934	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6935	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6936	}
6937	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6938	NOREF(idCpu);
6939
6940	return VINF_SUCCESS;
6941	}
6942
6943
6944	/**
6945	* Leaves the VT-x session.
6946	*
6947	* @returns VBox status code.
6948	* @param pVCpu The cross context virtual CPU structure.
6949	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6950	* out-of-sync. Make sure to update the required fields
6951	* before using them.
6952	*
6953	* @remarks No-long-jmp zone!!!
6954	*/
6955	static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6956	{
6957	HM_DISABLE_PREEMPT();
6958	HMVMX_ASSERT_CPU_SAFE();
6959	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6960	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6961
6962	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6963	and done this from the VMXR0ThreadCtxCallback(). */
6964	if (!pVCpu->hm.s.fLeaveDone)
6965	{
6966	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6967	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6968	pVCpu->hm.s.fLeaveDone = true;
6969	}
6970	Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6971
6972	/*
6973	* !!! IMPORTANT !!!
6974	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6975	*/
6976
6977	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6978	/** @todo Deregistering here means we need to VMCLEAR always
6979	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6980	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6981	VMMR0ThreadCtxHookDisable(pVCpu);
6982
6983	/* Leave HM context. This takes care of local init (term). */
6984	int rc = HMR0LeaveCpu(pVCpu);
6985
6986	HM_RESTORE_PREEMPT();
6987	return rc;
6988	}
6989
6990
6991	/**
6992	* Does the necessary state syncing before doing a longjmp to ring-3.
6993	*
6994	* @returns VBox status code.
6995	* @param pVCpu The cross context virtual CPU structure.
6996	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6997	* out-of-sync. Make sure to update the required fields
6998	* before using them.
6999	*
7000	* @remarks No-long-jmp zone!!!
7001	*/
7002	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7003	{
7004	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7005	}
7006
7007
7008	/**
7009	* Take necessary actions before going back to ring-3.
7010	*
7011	* An action requires us to go back to ring-3. This function does the necessary
7012	* steps before we can safely return to ring-3. This is not the same as longjmps
7013	* to ring-3, this is voluntary and prepares the guest so it may continue
7014	* executing outside HM (recompiler/IEM).
7015	*
7016	* @returns VBox status code.
7017	* @param pVCpu The cross context virtual CPU structure.
7018	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7019	* out-of-sync. Make sure to update the required fields
7020	* before using them.
7021	* @param rcExit The reason for exiting to ring-3. Can be
7022	* VINF_VMM_UNKNOWN_RING3_CALL.
7023	*/
7024	static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7025	{
7026	Assert(pVCpu);
7027	Assert(pMixedCtx);
7028	HMVMX_ASSERT_PREEMPT_SAFE();
7029
7030	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7031	{
7032	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7033	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7034	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7035	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7036	}
7037
7038	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7039	VMMRZCallRing3Disable(pVCpu);
7040	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7041
7042	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7043	if (pVCpu->hm.s.Event.fPending)
7044	{
7045	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7046	Assert(!pVCpu->hm.s.Event.fPending);
7047	}
7048
7049	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7050	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7051
7052	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7053	and if we're injecting an event we should have a TRPM trap pending. */
7054	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7055	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7056	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7057	#endif
7058
7059	/* Save guest state and restore host state bits. */
7060	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7061	AssertRCReturn(rc, rc);
7062	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7063	/* Thread-context hooks are unregistered at this point!!! */
7064
7065	/* Sync recompiler state. */
7066	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7067	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7068	\| CPUM_CHANGED_LDTR
7069	\| CPUM_CHANGED_GDTR
7070	\| CPUM_CHANGED_IDTR
7071	\| CPUM_CHANGED_TR
7072	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7073	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7074	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7075	{
7076	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7077	}
7078
7079	Assert(!pVCpu->hm.s.fClearTrapFlag);
7080
7081	/* Update the exit-to-ring 3 reason. */
7082	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7083
7084	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7085	if (rcExit != VINF_EM_RAW_INTERRUPT)
7086	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7087
7088	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7089
7090	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7091	VMMRZCallRing3RemoveNotification(pVCpu);
7092	VMMRZCallRing3Enable(pVCpu);
7093
7094	return rc;
7095	}
7096
7097
7098	/**
7099	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7100	* longjump to ring-3 and possibly get preempted.
7101	*
7102	* @returns VBox status code.
7103	* @param pVCpu The cross context virtual CPU structure.
7104	* @param enmOperation The operation causing the ring-3 longjump.
7105	* @param pvUser Opaque pointer to the guest-CPU context. The data
7106	* may be out-of-sync. Make sure to update the required
7107	* fields before using them.
7108	*/
7109	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7110	{
7111	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7112	{
7113	/*
7114	* !!! IMPORTANT !!!
7115	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7116	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7117	*/
7118	VMMRZCallRing3RemoveNotification(pVCpu);
7119	VMMRZCallRing3Disable(pVCpu);
7120	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7121	RTThreadPreemptDisable(&PreemptState);
7122
7123	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7124	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7125
7126	#if HC_ARCH_BITS == 64
7127	/* Restore host-state bits that VT-x only restores partially. */
7128	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7129	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7130	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7131	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7132	#endif
7133
7134	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7135	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7136	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7137
7138	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7139	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7140	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7141	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7142	{
7143	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7144	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7145	}
7146
7147	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7148	VMMR0ThreadCtxHookDisable(pVCpu);
7149	HMR0LeaveCpu(pVCpu);
7150	RTThreadPreemptRestore(&PreemptState);
7151	return VINF_SUCCESS;
7152	}
7153
7154	Assert(pVCpu);
7155	Assert(pvUser);
7156	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7157	HMVMX_ASSERT_PREEMPT_SAFE();
7158
7159	VMMRZCallRing3Disable(pVCpu);
7160	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7161
7162	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7163
7164	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7165	AssertRCReturn(rc, rc);
7166
7167	VMMRZCallRing3Enable(pVCpu);
7168	return VINF_SUCCESS;
7169	}
7170
7171
7172	/**
7173	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7174	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7175	*
7176	* @param pVCpu The cross context virtual CPU structure.
7177	*/
7178	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7179	{
7180	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7181	{
7182	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7183	{
7184	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7185	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7186	AssertRC(rc);
7187	Log4Func(("Setup interrupt-window exiting\n"));
7188	}
7189	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7190	}
7191
7192
7193	/**
7194	* Clears the interrupt-window exiting control in the VMCS.
7195	*
7196	* @param pVCpu The cross context virtual CPU structure.
7197	*/
7198	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7199	{
7200	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7201	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7202	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7203	AssertRC(rc);
7204	Log4Func(("Cleared interrupt-window exiting\n"));
7205	}
7206
7207
7208	/**
7209	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7210	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7211	*
7212	* @param pVCpu The cross context virtual CPU structure.
7213	*/
7214	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7215	{
7216	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7217	{
7218	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7219	{
7220	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7221	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7222	AssertRC(rc);
7223	Log4Func(("Setup NMI-window exiting\n"));
7224	}
7225	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7226	}
7227
7228
7229	/**
7230	* Clears the NMI-window exiting control in the VMCS.
7231	*
7232	* @param pVCpu The cross context virtual CPU structure.
7233	*/
7234	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7235	{
7236	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7237	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7238	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7239	AssertRC(rc);
7240	Log4Func(("Cleared NMI-window exiting\n"));
7241	}
7242
7243
7244	/**
7245	* Evaluates the event to be delivered to the guest and sets it as the pending
7246	* event.
7247	*
7248	* @returns The VT-x guest-interruptibility state.
7249	* @param pVCpu The cross context virtual CPU structure.
7250	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7251	* out-of-sync. Make sure to update the required fields
7252	* before using them.
7253	*/
7254	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7255	{
7256	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7257	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7258	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7259	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7260	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7261
7262	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7263	Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7264	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7265	Assert(!TRPMHasTrap(pVCpu));
7266
7267	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7268	APICUpdatePendingInterrupts(pVCpu);
7269
7270	/*
7271	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7272	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7273	*/
7274	/** @todo SMI. SMIs take priority over NMIs. */
7275	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7276	{
7277	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7278	if ( !pVCpu->hm.s.Event.fPending
7279	&& !fBlockNmi
7280	&& !fBlockSti
7281	&& !fBlockMovSS)
7282	{
7283	Log4Func(("Pending NMI\n"));
7284	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7285	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7286
7287	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7288	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7289	}
7290	else
7291	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7292	}
7293	/*
7294	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7295	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7296	*/
7297	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7298	&& !pVCpu->hm.s.fSingleInstruction)
7299	{
7300	Assert(!DBGFIsStepping(pVCpu));
7301	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7302	AssertRCReturn(rc, 0);
7303	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7304	if ( !pVCpu->hm.s.Event.fPending
7305	&& !fBlockInt
7306	&& !fBlockSti
7307	&& !fBlockMovSS)
7308	{
7309	uint8_t u8Interrupt;
7310	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7311	if (RT_SUCCESS(rc))
7312	{
7313	Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7314	uint32_t u32IntInfo = u8Interrupt
7315	\| VMX_EXIT_INTERRUPTION_INFO_VALID
7316	\| (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7317
7318	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7319	}
7320	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7321	{
7322	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7323	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7324	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7325
7326	/*
7327	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7328	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7329	* need to re-set this force-flag here.
7330	*/
7331	}
7332	else
7333	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7334	}
7335	else
7336	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7337	}
7338
7339	return fIntrState;
7340	}
7341
7342
7343	/**
7344	* Sets a pending-debug exception to be delivered to the guest if the guest is
7345	* single-stepping in the VMCS.
7346	*
7347	* @param pVCpu The cross context virtual CPU structure.
7348	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7349	* out-of-sync. Make sure to update the required fields
7350	* before using them.
7351	*/
7352	DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7353	{
7354	RT_NOREF(pVCpu);
7355	Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7356	return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7357	}
7358
7359
7360	/**
7361	* Injects any pending events into the guest if the guest is in a state to
7362	* receive them.
7363	*
7364	* @returns Strict VBox status code (i.e. informational status codes too).
7365	* @param pVCpu The cross context virtual CPU structure.
7366	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7367	* out-of-sync. Make sure to update the required fields
7368	* before using them.
7369	* @param fIntrState The VT-x guest-interruptibility state.
7370	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7371	* return VINF_EM_DBG_STEPPED if the event was
7372	* dispatched directly.
7373	*/
7374	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7375	{
7376	HMVMX_ASSERT_PREEMPT_SAFE();
7377	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7378
7379	bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7380	bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7381
7382	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7383	Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7384	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7385	Assert(!TRPMHasTrap(pVCpu));
7386
7387	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7388	if (pVCpu->hm.s.Event.fPending)
7389	{
7390	/*
7391	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7392	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7393	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7394	*
7395	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7396	*/
7397	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7398	#ifdef VBOX_STRICT
7399	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7400	{
7401	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7402	Assert(!fBlockInt);
7403	Assert(!fBlockSti);
7404	Assert(!fBlockMovSS);
7405	}
7406	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7407	{
7408	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7409	Assert(!fBlockSti);
7410	Assert(!fBlockMovSS);
7411	Assert(!fBlockNmi);
7412	}
7413	#endif
7414	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7415	uIntType));
7416	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7417	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7418	&fIntrState);
7419	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7420
7421	/* Update the interruptibility-state as it could have been changed by
7422	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7423	fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7424	fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7425
7426	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7427	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7428	else
7429	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7430	}
7431
7432	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7433	if ( fBlockSti
7434	\|\| fBlockMovSS)
7435	{
7436	if (!pVCpu->hm.s.fSingleInstruction)
7437	{
7438	/*
7439	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7440	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7441	* See Intel spec. 27.3.4 "Saving Non-Register State".
7442	*/
7443	Assert(!DBGFIsStepping(pVCpu));
7444	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7445	AssertRCReturn(rc, rc);
7446	if (pMixedCtx->eflags.Bits.u1TF)
7447	{
7448	int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7449	AssertRCReturn(rc2, rc2);
7450	}
7451	}
7452	else if (pMixedCtx->eflags.Bits.u1TF)
7453	{
7454	/*
7455	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7456	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7457	*/
7458	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7459	fIntrState = 0;
7460	}
7461	}
7462
7463	/*
7464	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7465	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7466	*/
7467	int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7468	AssertRCReturn(rc3, rc3);
7469
7470	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7471	NOREF(fBlockMovSS); NOREF(fBlockSti);
7472	return rcStrict;
7473	}
7474
7475
7476	/**
7477	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7478	*
7479	* @param pVCpu The cross context virtual CPU structure.
7480	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7481	* out-of-sync. Make sure to update the required fields
7482	* before using them.
7483	*/
7484	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7485	{
7486	NOREF(pMixedCtx);
7487	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7488	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7489	}
7490
7491
7492	/**
7493	* Injects a double-fault (\#DF) exception into the VM.
7494	*
7495	* @returns Strict VBox status code (i.e. informational status codes too).
7496	* @param pVCpu The cross context virtual CPU structure.
7497	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7498	* out-of-sync. Make sure to update the required fields
7499	* before using them.
7500	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7501	* and should return VINF_EM_DBG_STEPPED if the event
7502	* is injected directly (register modified by us, not
7503	* by hardware on VM-entry).
7504	* @param pfIntrState Pointer to the current guest interruptibility-state.
7505	* This interruptibility-state will be updated if
7506	* necessary. This cannot not be NULL.
7507	*/
7508	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7509	{
7510	NOREF(pMixedCtx);
7511	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7512	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7513	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7514	return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */, fStepping,
7515	pfIntrState);
7516	}
7517
7518
7519	/**
7520	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7521	*
7522	* @param pVCpu The cross context virtual CPU structure.
7523	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7524	* out-of-sync. Make sure to update the required fields
7525	* before using them.
7526	*/
7527	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7528	{
7529	NOREF(pMixedCtx);
7530	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7531	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7532	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7533	}
7534
7535
7536	/**
7537	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7538	*
7539	* @param pVCpu The cross context virtual CPU structure.
7540	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7541	* out-of-sync. Make sure to update the required fields
7542	* before using them.
7543	* @param cbInstr The value of RIP that is to be pushed on the guest
7544	* stack.
7545	*/
7546	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7547	{
7548	NOREF(pMixedCtx);
7549	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7550	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7551	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7552	}
7553
7554
7555	/**
7556	* Injects a general-protection (\#GP) fault into the VM.
7557	*
7558	* @returns Strict VBox status code (i.e. informational status codes too).
7559	* @param pVCpu The cross context virtual CPU structure.
7560	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7561	* out-of-sync. Make sure to update the required fields
7562	* before using them.
7563	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7564	* mode, i.e. in real-mode it's not valid).
7565	* @param u32ErrorCode The error code associated with the \#GP.
7566	* @param fStepping Whether we're running in
7567	* hmR0VmxRunGuestCodeStep() and should return
7568	* VINF_EM_DBG_STEPPED if the event is injected
7569	* directly (register modified by us, not by
7570	* hardware on VM-entry).
7571	* @param pfIntrState Pointer to the current guest interruptibility-state.
7572	* This interruptibility-state will be updated if
7573	* necessary. This cannot not be NULL.
7574	*/
7575	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7576	bool fStepping, uint32_t *pfIntrState)
7577	{
7578	NOREF(pMixedCtx);
7579	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7580	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7581	if (fErrorCodeValid)
7582	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7583	return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */, fStepping,
7584	pfIntrState);
7585	}
7586
7587
7588	#if 0 /* unused */
7589	/**
7590	* Sets a general-protection (\#GP) exception as pending-for-injection into the
7591	* VM.
7592	*
7593	* @param pVCpu The cross context virtual CPU structure.
7594	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7595	* out-of-sync. Make sure to update the required fields
7596	* before using them.
7597	* @param u32ErrorCode The error code associated with the \#GP.
7598	*/
7599	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7600	{
7601	NOREF(pMixedCtx);
7602	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7603	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7604	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7605	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
7606	}
7607	#endif /* unused */
7608
7609
7610	/**
7611	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7612	*
7613	* @param pVCpu The cross context virtual CPU structure.
7614	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7615	* out-of-sync. Make sure to update the required fields
7616	* before using them.
7617	* @param uVector The software interrupt vector number.
7618	* @param cbInstr The value of RIP that is to be pushed on the guest
7619	* stack.
7620	*/
7621	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7622	{
7623	NOREF(pMixedCtx);
7624	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7625	if ( uVector == X86_XCPT_BP
7626	\|\| uVector == X86_XCPT_OF)
7627	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7628	else
7629	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7630	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7631	}
7632
7633
7634	/**
7635	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7636	* stack.
7637	*
7638	* @returns Strict VBox status code (i.e. informational status codes too).
7639	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7640	* @param pVM The cross context VM structure.
7641	* @param pMixedCtx Pointer to the guest-CPU context.
7642	* @param uValue The value to push to the guest stack.
7643	*/
7644	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7645	{
7646	/*
7647	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7648	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7649	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7650	*/
7651	if (pMixedCtx->sp == 1)
7652	return VINF_EM_RESET;
7653	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7654	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7655	AssertRC(rc);
7656	return rc;
7657	}
7658
7659
7660	/**
7661	* Injects an event into the guest upon VM-entry by updating the relevant fields
7662	* in the VM-entry area in the VMCS.
7663	*
7664	* @returns Strict VBox status code (i.e. informational status codes too).
7665	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7666	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7667	*
7668	* @param pVCpu The cross context virtual CPU structure.
7669	* @param u64IntInfo The VM-entry interruption-information field.
7670	* @param cbInstr The VM-entry instruction length in bytes (for
7671	* software interrupts, exceptions and privileged
7672	* software exceptions).
7673	* @param u32ErrCode The VM-entry exception error code.
7674	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7675	* @param pfIntrState Pointer to the current guest interruptibility-state.
7676	* This interruptibility-state will be updated if
7677	* necessary. This cannot not be NULL.
7678	* @param fStepping Whether we're running in
7679	* hmR0VmxRunGuestCodeStep() and should return
7680	* VINF_EM_DBG_STEPPED if the event is injected
7681	* directly (register modified by us, not by
7682	* hardware on VM-entry).
7683	*/
7684	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7685	RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7686	{
7687	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7688	AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7689	Assert(pfIntrState);
7690
7691	PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7692	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7693	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7694	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7695
7696	#ifdef VBOX_STRICT
7697	/*
7698	* Validate the error-code-valid bit for hardware exceptions.
7699	* No error codes for exceptions in real-mode.
7700	*
7701	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7702	*/
7703	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7704	&& !CPUMIsGuestInRealModeEx(pMixedCtx))
7705	{
7706	switch (uVector)
7707	{
7708	case X86_XCPT_PF:
7709	case X86_XCPT_DF:
7710	case X86_XCPT_TS:
7711	case X86_XCPT_NP:
7712	case X86_XCPT_SS:
7713	case X86_XCPT_GP:
7714	case X86_XCPT_AC:
7715	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7716	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7717	RT_FALL_THRU();
7718	default:
7719	break;
7720	}
7721	}
7722	#endif
7723
7724	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7725	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7726	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7727
7728	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7729
7730	/*
7731	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
7732	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7733	* interrupt handler in the (real-mode) guest.
7734	*
7735	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7736	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7737	*/
7738	if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7739	{
7740	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7741	{
7742	/*
7743	* For unrestricted execution enabled CPUs running real-mode guests, we must not
7744	* set the deliver-error-code bit.
7745	*
7746	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7747	*/
7748	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7749	}
7750	else
7751	{
7752	PVM pVM = pVCpu->CTX_SUFF(pVM);
7753	Assert(PDMVmmDevHeapIsEnabled(pVM));
7754	Assert(pVM->hm.s.vmx.pRealModeTSS);
7755
7756	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7757	int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7758	\| CPUMCTX_EXTRN_TABLE_MASK
7759	\| CPUMCTX_EXTRN_RIP
7760	\| CPUMCTX_EXTRN_RSP
7761	\| CPUMCTX_EXTRN_RFLAGS);
7762	AssertRCReturn(rc2, rc2);
7763
7764	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7765	size_t const cbIdtEntry = sizeof(X86IDTR16);
7766	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7767	{
7768	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7769	if (uVector == X86_XCPT_DF)
7770	return VINF_EM_RESET;
7771
7772	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7773	if (uVector == X86_XCPT_GP)
7774	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7775
7776	/*
7777	* If we're injecting an event with no valid IDT entry, inject a #GP.
7778	* No error codes for exceptions in real-mode.
7779	*
7780	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7781	*/
7782	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, fStepping,
7783	pfIntrState);
7784	}
7785
7786	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7787	uint16_t uGuestIp = pMixedCtx->ip;
7788	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7789	{
7790	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7791	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7792	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7793	}
7794	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7795	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7796
7797	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7798	X86IDTR16 IdtEntry;
7799	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7800	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7801	AssertRCReturn(rc2, rc2);
7802
7803	/* Construct the stack frame for the interrupt/exception handler. */
7804	VBOXSTRICTRC rcStrict;
7805	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7806	if (rcStrict == VINF_SUCCESS)
7807	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7808	if (rcStrict == VINF_SUCCESS)
7809	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7810
7811	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7812	if (rcStrict == VINF_SUCCESS)
7813	{
7814	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7815	pMixedCtx->rip = IdtEntry.offSel;
7816	pMixedCtx->cs.Sel = IdtEntry.uSel;
7817	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7818	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7819	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7820	&& uVector == X86_XCPT_PF)
7821	pMixedCtx->cr2 = GCPtrFaultAddress;
7822
7823	/* If any other guest-state bits are changed here, make sure to update
7824	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7825	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7826	\| HM_CHANGED_GUEST_CR2
7827	\| HM_CHANGED_GUEST_RIP
7828	\| HM_CHANGED_GUEST_RFLAGS
7829	\| HM_CHANGED_GUEST_RSP);
7830
7831	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7832	if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7833	{
7834	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7835	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7836	Log4Func(("Clearing inhibition due to STI\n"));
7837	*pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7838	}
7839	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7840	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7841
7842	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7843	it, if we are returning to ring-3 before executing guest code. */
7844	pVCpu->hm.s.Event.fPending = false;
7845
7846	/* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7847	if (fStepping)
7848	rcStrict = VINF_EM_DBG_STEPPED;
7849	}
7850	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7851	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7852	return rcStrict;
7853	}
7854	}
7855
7856	/* Validate. */
7857	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7858	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7859	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7860
7861	/* Inject. */
7862	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7863	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7864	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7865	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7866	AssertRCReturn(rc, rc);
7867
7868	/* Update CR2. */
7869	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7870	&& uVector == X86_XCPT_PF)
7871	pMixedCtx->cr2 = GCPtrFaultAddress;
7872
7873	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7874
7875	return VINF_SUCCESS;
7876	}
7877
7878
7879	/**
7880	* Clears the interrupt-window exiting control in the VMCS and if necessary
7881	* clears the current event in the VMCS as well.
7882	*
7883	* @returns VBox status code.
7884	* @param pVCpu The cross context virtual CPU structure.
7885	*
7886	* @remarks Use this function only to clear events that have not yet been
7887	* delivered to the guest but are injected in the VMCS!
7888	* @remarks No-long-jump zone!!!
7889	*/
7890	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7891	{
7892	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7893	{
7894	hmR0VmxClearIntWindowExitVmcs(pVCpu);
7895	Log4Func(("Cleared interrupt widow\n"));
7896	}
7897
7898	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7899	{
7900	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7901	Log4Func(("Cleared interrupt widow\n"));
7902	}
7903	}
7904
7905
7906	/**
7907	* Enters the VT-x session.
7908	*
7909	* @returns VBox status code.
7910	* @param pVCpu The cross context virtual CPU structure.
7911	* @param pHostCpu Pointer to the global CPU info struct.
7912	*/
7913	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7914	{
7915	AssertPtr(pVCpu);
7916	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7917	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7918	RT_NOREF(pHostCpu);
7919
7920	LogFlowFunc(("pVCpu=%p\n", pVCpu));
7921	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7922	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7923
7924	#ifdef VBOX_STRICT
7925	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7926	RTCCUINTREG uHostCR4 = ASMGetCR4();
7927	if (!(uHostCR4 & X86_CR4_VMXE))
7928	{
7929	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7930	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7931	}
7932	#endif
7933
7934	/*
7935	* Load the VCPU's VMCS as the current (and active) one.
7936	*/
7937	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7938	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7939	if (RT_FAILURE(rc))
7940	return rc;
7941
7942	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7943	pVCpu->hm.s.fLeaveDone = false;
7944	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7945
7946	return VINF_SUCCESS;
7947	}
7948
7949
7950	/**
7951	* The thread-context callback (only on platforms which support it).
7952	*
7953	* @param enmEvent The thread-context event.
7954	* @param pVCpu The cross context virtual CPU structure.
7955	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7956	* @thread EMT(pVCpu)
7957	*/
7958	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7959	{
7960	NOREF(fGlobalInit);
7961
7962	switch (enmEvent)
7963	{
7964	case RTTHREADCTXEVENT_OUT:
7965	{
7966	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7967	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7968	VMCPU_ASSERT_EMT(pVCpu);
7969
7970	/* No longjmps (logger flushes, locks) in this fragile context. */
7971	VMMRZCallRing3Disable(pVCpu);
7972	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7973
7974	/*
7975	* Restore host-state (FPU, debug etc.)
7976	*/
7977	if (!pVCpu->hm.s.fLeaveDone)
7978	{
7979	/*
7980	* Do -not- import the guest-state here as we might already be in the middle of importing
7981	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7982	*/
7983	hmR0VmxLeave(pVCpu, false /* fImportState */);
7984	pVCpu->hm.s.fLeaveDone = true;
7985	}
7986
7987	/* Leave HM context, takes care of local init (term). */
7988	int rc = HMR0LeaveCpu(pVCpu);
7989	AssertRC(rc); NOREF(rc);
7990
7991	/* Restore longjmp state. */
7992	VMMRZCallRing3Enable(pVCpu);
7993	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7994	break;
7995	}
7996
7997	case RTTHREADCTXEVENT_IN:
7998	{
7999	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8000	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8001	VMCPU_ASSERT_EMT(pVCpu);
8002
8003	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8004	VMMRZCallRing3Disable(pVCpu);
8005	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8006
8007	/* Initialize the bare minimum state required for HM. This takes care of
8008	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8009	int rc = hmR0EnterCpu(pVCpu);
8010	AssertRC(rc);
8011	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8012	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8013
8014	/* Load the active VMCS as the current one. */
8015	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8016	{
8017	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8018	AssertRC(rc); NOREF(rc);
8019	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8020	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8021	}
8022	pVCpu->hm.s.fLeaveDone = false;
8023
8024	/* Restore longjmp state. */
8025	VMMRZCallRing3Enable(pVCpu);
8026	break;
8027	}
8028
8029	default:
8030	break;
8031	}
8032	}
8033
8034
8035	/**
8036	* Exports the host state into the VMCS host-state area.
8037	* Sets up the VM-exit MSR-load area.
8038	*
8039	* The CPU state will be loaded from these fields on every successful VM-exit.
8040	*
8041	* @returns VBox status code.
8042	* @param pVCpu The cross context virtual CPU structure.
8043	*
8044	* @remarks No-long-jump zone!!!
8045	*/
8046	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8047	{
8048	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8049
8050	int rc = VINF_SUCCESS;
8051	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8052	{
8053	rc = hmR0VmxExportHostControlRegs();
8054	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8055
8056	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8057	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8058
8059	rc = hmR0VmxExportHostMsrs(pVCpu);
8060	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8061
8062	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8063	}
8064	return rc;
8065	}
8066
8067
8068	/**
8069	* Saves the host state in the VMCS host-state.
8070	*
8071	* @returns VBox status code.
8072	* @param pVCpu The cross context virtual CPU structure.
8073	*
8074	* @remarks No-long-jump zone!!!
8075	*/
8076	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8077	{
8078	AssertPtr(pVCpu);
8079	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8080
8081	/*
8082	* Export the host state here while entering HM context.
8083	* When thread-context hooks are used, we might get preempted and have to re-save the host
8084	* state but most of the time we won't be, so do it here before we disable interrupts.
8085	*/
8086	return hmR0VmxExportHostState(pVCpu);
8087	}
8088
8089
8090	/**
8091	* Exports the guest state into the VMCS guest-state area.
8092	*
8093	* The will typically be done before VM-entry when the guest-CPU state and the
8094	* VMCS state may potentially be out of sync.
8095	*
8096	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8097	* VM-entry controls.
8098	* Sets up the appropriate VMX non-root function to execute guest code based on
8099	* the guest CPU mode.
8100	*
8101	* @returns VBox strict status code.
8102	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8103	* without unrestricted guest access and the VMMDev is not presently
8104	* mapped (e.g. EFI32).
8105	*
8106	* @param pVCpu The cross context virtual CPU structure.
8107	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8108	* out-of-sync. Make sure to update the required fields
8109	* before using them.
8110	*
8111	* @remarks No-long-jump zone!!!
8112	*/
8113	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8114	{
8115	AssertPtr(pVCpu);
8116	AssertPtr(pMixedCtx);
8117	HMVMX_ASSERT_PREEMPT_SAFE();
8118
8119	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8120
8121	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8122
8123	/* Determine real-on-v86 mode. */
8124	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8125	if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8126	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8127	{
8128	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8129	}
8130
8131	/*
8132	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8133	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8134	*/
8135	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8136	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8137
8138	/* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8139	rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8140	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8141
8142	/* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8143	rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8144	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8145
8146	rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8147	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8148
8149	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8150	if (rcStrict == VINF_SUCCESS)
8151	{ /* likely */ }
8152	else
8153	{
8154	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8155	return rcStrict;
8156	}
8157
8158	rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8159	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8160
8161	/* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8162	may alter controls if we determine we don't have to swap EFER after all. */
8163	rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8164	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8165
8166	rc = hmR0VmxExportGuestApicTpr(pVCpu);
8167	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8168
8169	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8170	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8171
8172	/* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8173	not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8174	rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8175	rc \|= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8176	rc \|= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8177	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8178
8179	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8180	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8181	\| HM_CHANGED_GUEST_CR2
8182	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8183	\| HM_CHANGED_GUEST_X87
8184	\| HM_CHANGED_GUEST_SSE_AVX
8185	\| HM_CHANGED_GUEST_OTHER_XSAVE
8186	\| HM_CHANGED_GUEST_XCRx
8187	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8188	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8189	\| HM_CHANGED_GUEST_TSC_AUX
8190	\| HM_CHANGED_GUEST_OTHER_MSRS
8191	\| HM_CHANGED_GUEST_HWVIRT
8192	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8193
8194	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8195	return rc;
8196	}
8197
8198
8199	/**
8200	* Exports the state shared between the host and guest into the VMCS.
8201	*
8202	* @param pVCpu The cross context virtual CPU structure.
8203	* @param pCtx Pointer to the guest-CPU context.
8204	*
8205	* @remarks No-long-jump zone!!!
8206	*/
8207	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8208	{
8209	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8210	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8211
8212	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8213	{
8214	int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8215	AssertRC(rc);
8216	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8217
8218	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8219	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8220	{
8221	rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8222	AssertRC(rc);
8223	}
8224	}
8225
8226	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8227	{
8228	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8229	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8230	}
8231
8232	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8233	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8234	}
8235
8236
8237	/**
8238	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8239	*
8240	* @returns Strict VBox status code (i.e. informational status codes too).
8241	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8242	* without unrestricted guest access and the VMMDev is not presently
8243	* mapped (e.g. EFI32).
8244	*
8245	* @param pVCpu The cross context virtual CPU structure.
8246	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8247	* out-of-sync. Make sure to update the required fields
8248	* before using them.
8249	*
8250	* @remarks No-long-jump zone!!!
8251	*/
8252	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8253	{
8254	HMVMX_ASSERT_PREEMPT_SAFE();
8255	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8256	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8257
8258	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8259	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8260	#endif
8261
8262	/*
8263	* For many exits it's only RIP that changes and hence try to export it first
8264	* without going through a lot of change flag checks.
8265	*/
8266	VBOXSTRICTRC rcStrict;
8267	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8268	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8269	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8270	{
8271	rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8272	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8273	{ /* likely */}
8274	else
8275	AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8276	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8277	}
8278	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8279	{
8280	rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8281	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8282	{ /* likely */}
8283	else
8284	{
8285	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8286	VBOXSTRICTRC_VAL(rcStrict)));
8287	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8288	return rcStrict;
8289	}
8290	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8291	}
8292	else
8293	rcStrict = VINF_SUCCESS;
8294
8295	#ifdef VBOX_STRICT
8296	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8297	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8298	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8299	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8300	("fCtxChanged=%#RX64\n", fCtxChanged));
8301	#endif
8302	return rcStrict;
8303	}
8304
8305
8306	/**
8307	* Does the preparations before executing guest code in VT-x.
8308	*
8309	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8310	* recompiler/IEM. We must be cautious what we do here regarding committing
8311	* guest-state information into the VMCS assuming we assuredly execute the
8312	* guest in VT-x mode.
8313	*
8314	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8315	* the common-state (TRPM/forceflags), we must undo those changes so that the
8316	* recompiler/IEM can (and should) use them when it resumes guest execution.
8317	* Otherwise such operations must be done when we can no longer exit to ring-3.
8318	*
8319	* @returns Strict VBox status code (i.e. informational status codes too).
8320	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8321	* have been disabled.
8322	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8323	* double-fault into the guest.
8324	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8325	* dispatched directly.
8326	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8327	*
8328	* @param pVCpu The cross context virtual CPU structure.
8329	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8330	* out-of-sync. Make sure to update the required fields
8331	* before using them.
8332	* @param pVmxTransient Pointer to the VMX transient structure.
8333	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8334	* us ignore some of the reasons for returning to
8335	* ring-3, and return VINF_EM_DBG_STEPPED if event
8336	* dispatching took place.
8337	*/
8338	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8339	{
8340	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8341
8342	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8343	PGMRZDynMapFlushAutoSet(pVCpu);
8344	#endif
8345
8346	/* Check force flag actions that might require us to go back to ring-3. */
8347	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8348	if (rcStrict == VINF_SUCCESS)
8349	{ /* FFs doesn't get set all the time. */ }
8350	else
8351	return rcStrict;
8352
8353	/*
8354	* Setup the virtualized-APIC accesses.
8355	*
8356	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8357	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8358	*
8359	* This is the reason we do it here and not in hmR0VmxExportGuestState().
8360	*/
8361	PVM pVM = pVCpu->CTX_SUFF(pVM);
8362	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8363	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8364	&& PDMHasApic(pVM))
8365	{
8366	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8367	Assert(u64MsrApicBase);
8368	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8369
8370	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8371
8372	/* Unalias any existing mapping. */
8373	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8374	AssertRCReturn(rc, rc);
8375
8376	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8377	Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8378	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8379	AssertRCReturn(rc, rc);
8380
8381	/* Update the per-VCPU cache of the APIC base MSR. */
8382	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8383	}
8384
8385	if (TRPMHasTrap(pVCpu))
8386	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8387	uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8388
8389	/*
8390	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8391	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8392	* also result in triple-faulting the VM.
8393	*/
8394	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8395	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8396	{ /* likely */ }
8397	else
8398	{
8399	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8400	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8401	return rcStrict;
8402	}
8403
8404	/*
8405	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
8406	* import CR3 themselves. We will need to update them here as even as late as the above
8407	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8408	* the below force flags to be set.
8409	*/
8410	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8411	{
8412	Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8413	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8414	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
8415	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8416	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8417	}
8418	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8419	{
8420	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8421	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8422	}
8423
8424	/*
8425	* No longjmps to ring-3 from this point on!!!
8426	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8427	* This also disables flushing of the R0-logger instance (if any).
8428	*/
8429	VMMRZCallRing3Disable(pVCpu);
8430
8431	/*
8432	* Export the guest state bits.
8433	*
8434	* We cannot perform longjmps while loading the guest state because we do not preserve the
8435	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8436	* CPU migration.
8437	*
8438	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8439	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8440	* Hence, loading of the guest state needs to be done -after- injection of events.
8441	*/
8442	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8443	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8444	{ /* likely */ }
8445	else
8446	{
8447	VMMRZCallRing3Enable(pVCpu);
8448	return rcStrict;
8449	}
8450
8451	/*
8452	* We disable interrupts so that we don't miss any interrupts that would flag preemption
8453	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8454	* preemption disabled for a while. Since this is purly to aid the
8455	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8456	* disable interrupt on NT.
8457	*
8458	* We need to check for force-flags that could've possible been altered since we last
8459	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8460	* see @bugref{6398}).
8461	*
8462	* We also check a couple of other force-flags as a last opportunity to get the EMT back
8463	* to ring-3 before executing guest code.
8464	*/
8465	pVmxTransient->fEFlags = ASMIntDisableFlags();
8466
8467	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8468	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8469	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8470	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8471	{
8472	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8473	{
8474	pVCpu->hm.s.Event.fPending = false;
8475
8476	/*
8477	* We've injected any pending events. This is really the point of no return (to ring-3).
8478	*
8479	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
8480	* returns from this function, so don't enable them here.
8481	*/
8482	return VINF_SUCCESS;
8483	}
8484
8485	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8486	rcStrict = VINF_EM_RAW_INTERRUPT;
8487	}
8488	else
8489	{
8490	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8491	rcStrict = VINF_EM_RAW_TO_R3;
8492	}
8493
8494	ASMSetFlags(pVmxTransient->fEFlags);
8495	VMMRZCallRing3Enable(pVCpu);
8496
8497	return rcStrict;
8498	}
8499
8500
8501	/**
8502	* Prepares to run guest code in VT-x and we've committed to doing so. This
8503	* means there is no backing out to ring-3 or anywhere else at this
8504	* point.
8505	*
8506	* @param pVCpu The cross context virtual CPU structure.
8507	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8508	* out-of-sync. Make sure to update the required fields
8509	* before using them.
8510	* @param pVmxTransient Pointer to the VMX transient structure.
8511	*
8512	* @remarks Called with preemption disabled.
8513	* @remarks No-long-jump zone!!!
8514	*/
8515	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8516	{
8517	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8518	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8519	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8520
8521	/*
8522	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8523	*/
8524	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8525	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8526
8527	PVM pVM = pVCpu->CTX_SUFF(pVM);
8528	if (!CPUMIsGuestFPUStateActive(pVCpu))
8529	{
8530	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8531	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8532	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
8533	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8534	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8535	}
8536
8537	/*
8538	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8539	*/
8540	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8541	&& pVCpu->hm.s.vmx.cMsrs > 0)
8542	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8543
8544	/*
8545	* Re-save the host state bits as we may've been preempted (only happens when
8546	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8547	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8548	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
8549	* See @bugref{8432}.
8550	*/
8551	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8552	{
8553	int rc = hmR0VmxExportHostState(pVCpu);
8554	AssertRC(rc);
8555	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8556	}
8557	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8558
8559	/*
8560	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
8561	*/
8562	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8563	hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8564	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8565
8566	/* Store status of the shared guest-host state at the time of VM-entry. */
8567	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8568	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8569	{
8570	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8571	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8572	}
8573	else
8574	#endif
8575	{
8576	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8577	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8578	}
8579
8580	/*
8581	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8582	*/
8583	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8584	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8585
8586	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8587	RTCPUID idCurrentCpu = pCpu->idCpu;
8588	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8589	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8590	{
8591	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8592	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8593	}
8594
8595	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8596	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8597	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8598	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8599
8600	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8601
8602	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8603	to start executing. */
8604
8605	/*
8606	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8607	*/
8608	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8609	{
8610	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8611	{
8612	bool fMsrUpdated;
8613	hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8614	int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8615	&fMsrUpdated);
8616	AssertRC(rc2);
8617	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8618	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8619	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8620	}
8621	else
8622	{
8623	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8624	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8625	}
8626	}
8627
8628	if (pVM->cpum.ro.GuestFeatures.fIbrs)
8629	{
8630	bool fMsrUpdated;
8631	hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8632	int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8633	&fMsrUpdated);
8634	AssertRC(rc2);
8635	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8636	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8637	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8638	}
8639
8640	#ifdef VBOX_STRICT
8641	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8642	hmR0VmxCheckHostEferMsr(pVCpu);
8643	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8644	#endif
8645	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8646	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8647	{
8648	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8649	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8650	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8651	}
8652	#endif
8653	}
8654
8655
8656	/**
8657	* Performs some essential restoration of state after running guest code in
8658	* VT-x.
8659	*
8660	* @param pVCpu The cross context virtual CPU structure.
8661	* @param pVmxTransient Pointer to the VMX transient structure.
8662	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8663	*
8664	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
8665	*
8666	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8667	* unconditionally when it is safe to do so.
8668	*/
8669	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8670	{
8671	uint64_t const uHostTsc = ASMReadTSC();
8672	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8673
8674	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8675	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8676	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8677	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8678	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8679	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8680
8681	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8682	TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8683
8684	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8685	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8686	Assert(!ASMIntAreEnabled());
8687	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8688
8689	#if HC_ARCH_BITS == 64
8690	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8691	#endif
8692	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8693	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8694	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8695	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8696	#else
8697	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8698	#endif
8699	#ifdef VBOX_STRICT
8700	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8701	#endif
8702	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8703
8704	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8705	uint32_t uExitReason;
8706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8707	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8708	AssertRC(rc);
8709	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8710	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8711
8712	if (rcVMRun == VINF_SUCCESS)
8713	{
8714	/*
8715	* Update the VM-exit history array here even if the VM-entry failed due to:
8716	* - Invalid guest state.
8717	* - MSR loading.
8718	* - Machine-check event.
8719	*
8720	* In any of the above cases we will still have a "valid" VM-exit reason
8721	* despite @a fVMEntryFailed being false.
8722	*
8723	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8724	*
8725	* Note! We don't have CS or RIP at this point. Will probably address that later
8726	* by amending the history entry added here.
8727	*/
8728	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8729	UINT64_MAX, uHostTsc);
8730
8731	if (!pVmxTransient->fVMEntryFailed)
8732	{
8733	VMMRZCallRing3Enable(pVCpu);
8734
8735	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8736	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8737
8738	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8739	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8740	AssertRC(rc);
8741	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8742	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8743	AssertRC(rc);
8744	#else
8745	/*
8746	* Import the guest-interruptibility state always as we need it while evaluating
8747	* injecting events on re-entry.
8748	*
8749	* We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8750	* checking for real-mode while exporting the state because all bits that cause
8751	* mode changes wrt CR0 are intercepted.
8752	*/
8753	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8754	AssertRC(rc);
8755	#endif
8756
8757	/*
8758	* Sync the TPR shadow with our APIC state.
8759	*/
8760	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8761	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8762	{
8763	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8764	AssertRC(rc);
8765	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8766	}
8767
8768	return;
8769	}
8770	}
8771	else
8772	{
8773	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8774	}
8775
8776	VMMRZCallRing3Enable(pVCpu);
8777	}
8778
8779
8780	/**
8781	* Runs the guest code using VT-x the normal way.
8782	*
8783	* @returns VBox status code.
8784	* @param pVCpu The cross context virtual CPU structure.
8785	* @param pCtx Pointer to the guest-CPU context.
8786	*
8787	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8788	*/
8789	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8790	{
8791	VMXTRANSIENT VmxTransient;
8792	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8793	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8794	uint32_t cLoops = 0;
8795
8796	for (;; cLoops++)
8797	{
8798	Assert(!HMR0SuspendPending());
8799	HMVMX_ASSERT_CPU_SAFE();
8800
8801	/* Preparatory work for running guest code, this may force us to return
8802	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8803	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8804	rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8805	if (rcStrict != VINF_SUCCESS)
8806	break;
8807
8808	hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8809	int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8810	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8811
8812	/* Restore any residual host-state and save any bits shared between host
8813	and guest into the guest-CPU state. Re-enables interrupts! */
8814	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8815
8816	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8817	if (RT_SUCCESS(rcRun))
8818	{ /* very likely */ }
8819	else
8820	{
8821	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8822	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8823	return rcRun;
8824	}
8825
8826	/* Profile the VM-exit. */
8827	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8828	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8829	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8830	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8831	HMVMX_START_EXIT_DISPATCH_PROF();
8832
8833	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8834
8835	/* Handle the VM-exit. */
8836	#ifdef HMVMX_USE_FUNCTION_TABLE
8837	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8838	#else
8839	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8840	#endif
8841	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8842	if (rcStrict == VINF_SUCCESS)
8843	{
8844	if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8845	continue; /* likely */
8846	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8847	rcStrict = VINF_EM_RAW_INTERRUPT;
8848	}
8849	break;
8850	}
8851
8852	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8853	return rcStrict;
8854	}
8855
8856
8857
8858	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8859	* probes.
8860	*
8861	* The following few functions and associated structure contains the bloat
8862	* necessary for providing detailed debug events and dtrace probes as well as
8863	* reliable host side single stepping. This works on the principle of
8864	* "subclassing" the normal execution loop and workers. We replace the loop
8865	* method completely and override selected helpers to add necessary adjustments
8866	* to their core operation.
8867	*
8868	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8869	* any performance for debug and analysis features.
8870	*
8871	* @{
8872	*/
8873
8874	/**
8875	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
8876	* the debug run loop.
8877	*/
8878	typedef struct VMXRUNDBGSTATE
8879	{
8880	/** The RIP we started executing at. This is for detecting that we stepped. */
8881	uint64_t uRipStart;
8882	/** The CS we started executing with. */
8883	uint16_t uCsStart;
8884
8885	/** Whether we've actually modified the 1st execution control field. */
8886	bool fModifiedProcCtls : 1;
8887	/** Whether we've actually modified the 2nd execution control field. */
8888	bool fModifiedProcCtls2 : 1;
8889	/** Whether we've actually modified the exception bitmap. */
8890	bool fModifiedXcptBitmap : 1;
8891
8892	/** We desire the modified the CR0 mask to be cleared. */
8893	bool fClearCr0Mask : 1;
8894	/** We desire the modified the CR4 mask to be cleared. */
8895	bool fClearCr4Mask : 1;
8896	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8897	uint32_t fCpe1Extra;
8898	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8899	uint32_t fCpe1Unwanted;
8900	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8901	uint32_t fCpe2Extra;
8902	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8903	uint32_t bmXcptExtra;
8904	/** The sequence number of the Dtrace provider settings the state was
8905	* configured against. */
8906	uint32_t uDtraceSettingsSeqNo;
8907	/** VM-exits to check (one bit per VM-exit). */
8908	uint32_t bmExitsToCheck[3];
8909
8910	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8911	uint32_t fProcCtlsInitial;
8912	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8913	uint32_t fProcCtls2Initial;
8914	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8915	uint32_t bmXcptInitial;
8916	} VMXRUNDBGSTATE;
8917	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8918	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8919
8920
8921	/**
8922	* Initializes the VMXRUNDBGSTATE structure.
8923	*
8924	* @param pVCpu The cross context virtual CPU structure of the
8925	* calling EMT.
8926	* @param pCtx The CPU register context to go with @a pVCpu.
8927	* @param pDbgState The structure to initialize.
8928	*/
8929	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8930	{
8931	pDbgState->uRipStart = pCtx->rip;
8932	pDbgState->uCsStart = pCtx->cs.Sel;
8933
8934	pDbgState->fModifiedProcCtls = false;
8935	pDbgState->fModifiedProcCtls2 = false;
8936	pDbgState->fModifiedXcptBitmap = false;
8937	pDbgState->fClearCr0Mask = false;
8938	pDbgState->fClearCr4Mask = false;
8939	pDbgState->fCpe1Extra = 0;
8940	pDbgState->fCpe1Unwanted = 0;
8941	pDbgState->fCpe2Extra = 0;
8942	pDbgState->bmXcptExtra = 0;
8943	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8944	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8945	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8946	}
8947
8948
8949	/**
8950	* Updates the VMSC fields with changes requested by @a pDbgState.
8951	*
8952	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8953	* immediately before executing guest code, i.e. when interrupts are disabled.
8954	* We don't check status codes here as we cannot easily assert or return in the
8955	* latter case.
8956	*
8957	* @param pVCpu The cross context virtual CPU structure.
8958	* @param pDbgState The debug state.
8959	*/
8960	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8961	{
8962	/*
8963	* Ensure desired flags in VMCS control fields are set.
8964	* (Ignoring write failure here, as we're committed and it's just debug extras.)
8965	*
8966	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8967	* there should be no stale data in pCtx at this point.
8968	*/
8969	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8970	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8971	{
8972	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
8973	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8974	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8975	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8976	pDbgState->fModifiedProcCtls = true;
8977	}
8978
8979	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8980	{
8981	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
8982	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8983	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8984	pDbgState->fModifiedProcCtls2 = true;
8985	}
8986
8987	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8988	{
8989	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
8990	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8991	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8992	pDbgState->fModifiedXcptBitmap = true;
8993	}
8994
8995	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8996	{
8997	pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8998	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8999	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9000	}
9001
9002	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9003	{
9004	pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9005	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9006	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9007	}
9008	}
9009
9010
9011	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9012	{
9013	/*
9014	* Restore VM-exit control settings as we may not reenter this function the
9015	* next time around.
9016	*/
9017	/* We reload the initial value, trigger what we can of recalculations the
9018	next time around. From the looks of things, that's all that's required atm. */
9019	if (pDbgState->fModifiedProcCtls)
9020	{
9021	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9022	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9023	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9024	AssertRCReturn(rc2, rc2);
9025	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9026	}
9027
9028	/* We're currently the only ones messing with this one, so just restore the
9029	cached value and reload the field. */
9030	if ( pDbgState->fModifiedProcCtls2
9031	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9032	{
9033	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9034	AssertRCReturn(rc2, rc2);
9035	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9036	}
9037
9038	/* If we've modified the exception bitmap, we restore it and trigger
9039	reloading and partial recalculation the next time around. */
9040	if (pDbgState->fModifiedXcptBitmap)
9041	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9042
9043	return rcStrict;
9044	}
9045
9046
9047	/**
9048	* Configures VM-exit controls for current DBGF and DTrace settings.
9049	*
9050	* This updates @a pDbgState and the VMCS execution control fields to reflect
9051	* the necessary VM-exits demanded by DBGF and DTrace.
9052	*
9053	* @param pVCpu The cross context virtual CPU structure.
9054	* @param pDbgState The debug state.
9055	* @param pVmxTransient Pointer to the VMX transient structure. May update
9056	* fUpdateTscOffsettingAndPreemptTimer.
9057	*/
9058	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9059	{
9060	/*
9061	* Take down the dtrace serial number so we can spot changes.
9062	*/
9063	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9064	ASMCompilerBarrier();
9065
9066	/*
9067	* We'll rebuild most of the middle block of data members (holding the
9068	* current settings) as we go along here, so start by clearing it all.
9069	*/
9070	pDbgState->bmXcptExtra = 0;
9071	pDbgState->fCpe1Extra = 0;
9072	pDbgState->fCpe1Unwanted = 0;
9073	pDbgState->fCpe2Extra = 0;
9074	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9075	pDbgState->bmExitsToCheck[i] = 0;
9076
9077	/*
9078	* Software interrupts (INT XXh) - no idea how to trigger these...
9079	*/
9080	PVM pVM = pVCpu->CTX_SUFF(pVM);
9081	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9082	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9083	{
9084	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9085	}
9086
9087	/*
9088	* INT3 breakpoints - triggered by #BP exceptions.
9089	*/
9090	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9091	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9092
9093	/*
9094	* Exception bitmap and XCPT events+probes.
9095	*/
9096	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9097	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9098	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9099
9100	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9101	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9102	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9103	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9104	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9105	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9106	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9107	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9108	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9109	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9110	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9111	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9112	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9113	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9114	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9115	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9116	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9117	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9118
9119	if (pDbgState->bmXcptExtra)
9120	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9121
9122	/*
9123	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9124	*
9125	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9126	* So, when adding/changing/removing please don't forget to update it.
9127	*
9128	* Some of the macros are picking up local variables to save horizontal space,
9129	* (being able to see it in a table is the lesser evil here).
9130	*/
9131	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9132	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9133	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9134	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9135	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9136	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9137	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9138	} else do { } while (0)
9139	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9140	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9141	{ \
9142	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9143	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9144	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9145	} else do { } while (0)
9146	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9147	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9148	{ \
9149	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9150	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9151	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9152	} else do { } while (0)
9153	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9154	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9155	{ \
9156	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9157	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9158	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9159	} else do { } while (0)
9160
9161	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9162	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9163	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9164	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9165	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9166
9167	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9168	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9169	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9170	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9171	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9172	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9173	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9174	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9175	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9176	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9177	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9178	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9179	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9180	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9181	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9182	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9183	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9184	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9185	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9186	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9187	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9188	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9189	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9190	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9191	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9192	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9193	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9194	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9195	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9196	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9197	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9198	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9199	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9200	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9201	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9202	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9203
9204	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9205	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9206	{
9207	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9208	\| CPUMCTX_EXTRN_CR4
9209	\| CPUMCTX_EXTRN_APIC_TPR);
9210	AssertRC(rc);
9211
9212	#if 0 /** @todo fix me */
9213	pDbgState->fClearCr0Mask = true;
9214	pDbgState->fClearCr4Mask = true;
9215	#endif
9216	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9217	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9218	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9219	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9220	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9221	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9222	require clearing here and in the loop if we start using it. */
9223	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9224	}
9225	else
9226	{
9227	if (pDbgState->fClearCr0Mask)
9228	{
9229	pDbgState->fClearCr0Mask = false;
9230	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9231	}
9232	if (pDbgState->fClearCr4Mask)
9233	{
9234	pDbgState->fClearCr4Mask = false;
9235	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9236	}
9237	}
9238	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9240
9241	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9242	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9243	{
9244	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9245	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9246	}
9247	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9248	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9249
9250	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9251	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9252	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9253	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9254	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9255	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9256	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9257	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9258	#if 0 /** @todo too slow, fix handler. */
9259	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9260	#endif
9261	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9262
9263	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9264	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9265	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9266	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9267	{
9268	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9269	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9270	}
9271	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9272	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9273	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9274	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9275
9276	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9277	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9278	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9279	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9280	{
9281	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9282	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9283	}
9284	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9285	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9286	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9287	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9288
9289	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9290	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9291	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9292	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9293	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9294	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9295	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9296	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9297	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9298	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9299	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9300	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9301	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9302	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9303	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9304	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9305	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9306	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9307	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9308	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9309	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9310	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9311
9312	#undef IS_EITHER_ENABLED
9313	#undef SET_ONLY_XBM_IF_EITHER_EN
9314	#undef SET_CPE1_XBM_IF_EITHER_EN
9315	#undef SET_CPEU_XBM_IF_EITHER_EN
9316	#undef SET_CPE2_XBM_IF_EITHER_EN
9317
9318	/*
9319	* Sanitize the control stuff.
9320	*/
9321	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9322	if (pDbgState->fCpe2Extra)
9323	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9324	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9325	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9326	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9327	{
9328	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9329	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9330	}
9331
9332	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9333	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9334	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9335	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9336	}
9337
9338
9339	/**
9340	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9341	* appropriate.
9342	*
9343	* The caller has checked the VM-exit against the
9344	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9345	* already, so we don't have to do that either.
9346	*
9347	* @returns Strict VBox status code (i.e. informational status codes too).
9348	* @param pVCpu The cross context virtual CPU structure.
9349	* @param pMixedCtx Pointer to the guest-CPU context.
9350	* @param pVmxTransient Pointer to the VMX-transient structure.
9351	* @param uExitReason The VM-exit reason.
9352	*
9353	* @remarks The name of this function is displayed by dtrace, so keep it short
9354	* and to the point. No longer than 33 chars long, please.
9355	*/
9356	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9357	uint32_t uExitReason)
9358	{
9359	/*
9360	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9361	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9362	*
9363	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9364	* does. Must add/change/remove both places. Same ordering, please.
9365	*
9366	* Added/removed events must also be reflected in the next section
9367	* where we dispatch dtrace events.
9368	*/
9369	bool fDtrace1 = false;
9370	bool fDtrace2 = false;
9371	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9372	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9373	uint32_t uEventArg = 0;
9374	#define SET_EXIT(a_EventSubName) \
9375	do { \
9376	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9377	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9378	} while (0)
9379	#define SET_BOTH(a_EventSubName) \
9380	do { \
9381	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9382	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9383	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9384	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9385	} while (0)
9386	switch (uExitReason)
9387	{
9388	case VMX_EXIT_MTF:
9389	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9390
9391	case VMX_EXIT_XCPT_OR_NMI:
9392	{
9393	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9394	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9395	{
9396	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9397	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9398	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9399	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9400	{
9401	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9402	{
9403	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9404	uEventArg = pVmxTransient->uExitIntErrorCode;
9405	}
9406	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9407	switch (enmEvent1)
9408	{
9409	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9410	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9411	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9412	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9413	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9414	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9415	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9416	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9417	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9418	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9419	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9420	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9421	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9422	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9423	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9424	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9425	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9426	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9427	default: break;
9428	}
9429	}
9430	else
9431	AssertFailed();
9432	break;
9433
9434	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9435	uEventArg = idxVector;
9436	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9437	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9438	break;
9439	}
9440	break;
9441	}
9442
9443	case VMX_EXIT_TRIPLE_FAULT:
9444	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9445	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9446	break;
9447	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9448	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9449	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9450	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9451	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9452
9453	/* Instruction specific VM-exits: */
9454	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9455	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9456	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9457	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9458	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9459	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9460	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9461	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9462	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9463	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9464	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9465	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9466	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9467	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9468	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9469	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9470	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9471	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9472	case VMX_EXIT_MOV_CRX:
9473	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9474	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9475	SET_BOTH(CRX_READ);
9476	else
9477	SET_BOTH(CRX_WRITE);
9478	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9479	break;
9480	case VMX_EXIT_MOV_DRX:
9481	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9482	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9483	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9484	SET_BOTH(DRX_READ);
9485	else
9486	SET_BOTH(DRX_WRITE);
9487	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9488	break;
9489	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9490	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9491	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9492	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9493	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9494	case VMX_EXIT_XDTR_ACCESS:
9495	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9496	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9497	{
9498	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9499	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9500	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9501	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9502	}
9503	break;
9504
9505	case VMX_EXIT_TR_ACCESS:
9506	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9507	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9508	{
9509	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9510	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9511	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9512	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9513	}
9514	break;
9515
9516	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9517	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9518	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9519	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9520	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9521	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9522	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9523	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9524	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9525	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9526	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9527
9528	/* Events that aren't relevant at this point. */
9529	case VMX_EXIT_EXT_INT:
9530	case VMX_EXIT_INT_WINDOW:
9531	case VMX_EXIT_NMI_WINDOW:
9532	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9533	case VMX_EXIT_PREEMPT_TIMER:
9534	case VMX_EXIT_IO_INSTR:
9535	break;
9536
9537	/* Errors and unexpected events. */
9538	case VMX_EXIT_INIT_SIGNAL:
9539	case VMX_EXIT_SIPI:
9540	case VMX_EXIT_IO_SMI:
9541	case VMX_EXIT_SMI:
9542	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9543	case VMX_EXIT_ERR_MSR_LOAD:
9544	case VMX_EXIT_ERR_MACHINE_CHECK:
9545	break;
9546
9547	default:
9548	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9549	break;
9550	}
9551	#undef SET_BOTH
9552	#undef SET_EXIT
9553
9554	/*
9555	* Dtrace tracepoints go first. We do them here at once so we don't
9556	* have to copy the guest state saving and stuff a few dozen times.
9557	* Down side is that we've got to repeat the switch, though this time
9558	* we use enmEvent since the probes are a subset of what DBGF does.
9559	*/
9560	if (fDtrace1 \|\| fDtrace2)
9561	{
9562	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9563	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9564	switch (enmEvent1)
9565	{
9566	/** @todo consider which extra parameters would be helpful for each probe. */
9567	case DBGFEVENT_END: break;
9568	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9569	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9570	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9571	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9572	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9573	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9574	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9575	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9576	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9577	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9578	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9579	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9580	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9581	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9582	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9583	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9584	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9585	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9586	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9587	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9588	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9589	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9590	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9591	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9592	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9593	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9594	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9595	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9596	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9597	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9598	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9599	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9600	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9601	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9602	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9603	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9604	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9605	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9606	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9607	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9608	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9609	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9610	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9611	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9612	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9613	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9614	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9615	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9616	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9617	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9618	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9619	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9620	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9621	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9622	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9623	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9624	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9625	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9626	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9627	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9628	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9629	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9630	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9631	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9632	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9633	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9634	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9635	}
9636	switch (enmEvent2)
9637	{
9638	/** @todo consider which extra parameters would be helpful for each probe. */
9639	case DBGFEVENT_END: break;
9640	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9641	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9642	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9643	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9644	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9645	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9646	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9647	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9648	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9649	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9650	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9651	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9652	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9653	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9654	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9655	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9656	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9657	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9658	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9659	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9660	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9661	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9662	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9663	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9664	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9665	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9666	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9667	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9668	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9669	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9670	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9671	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9672	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9673	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9674	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9675	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9676	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9677	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9678	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9679	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9680	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9681	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9682	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9683	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9684	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9685	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9686	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9687	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9688	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9689	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9690	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9691	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9692	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9693	}
9694	}
9695
9696	/*
9697	* Fire of the DBGF event, if enabled (our check here is just a quick one,
9698	* the DBGF call will do a full check).
9699	*
9700	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9701	* Note! If we have to events, we prioritize the first, i.e. the instruction
9702	* one, in order to avoid event nesting.
9703	*/
9704	PVM pVM = pVCpu->CTX_SUFF(pVM);
9705	if ( enmEvent1 != DBGFEVENT_END
9706	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9707	{
9708	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9709	if (rcStrict != VINF_SUCCESS)
9710	return rcStrict;
9711	}
9712	else if ( enmEvent2 != DBGFEVENT_END
9713	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9714	{
9715	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9716	if (rcStrict != VINF_SUCCESS)
9717	return rcStrict;
9718	}
9719
9720	return VINF_SUCCESS;
9721	}
9722
9723
9724	/**
9725	* Single-stepping VM-exit filtering.
9726	*
9727	* This is preprocessing the VM-exits and deciding whether we've gotten far
9728	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9729	* handling is performed.
9730	*
9731	* @returns Strict VBox status code (i.e. informational status codes too).
9732	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9733	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9734	* out-of-sync. Make sure to update the required
9735	* fields before using them.
9736	* @param pVmxTransient Pointer to the VMX-transient structure.
9737	* @param uExitReason The VM-exit reason.
9738	* @param pDbgState The debug state.
9739	*/
9740	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9741	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9742	{
9743	/*
9744	* Expensive (saves context) generic dtrace VM-exit probe.
9745	*/
9746	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9747	{ /* more likely */ }
9748	else
9749	{
9750	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9751	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9752	AssertRC(rc);
9753	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9754	}
9755
9756	/*
9757	* Check for host NMI, just to get that out of the way.
9758	*/
9759	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9760	{ /* normally likely */ }
9761	else
9762	{
9763	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9764	AssertRCReturn(rc2, rc2);
9765	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9766	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9767	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9768	}
9769
9770	/*
9771	* Check for single stepping event if we're stepping.
9772	*/
9773	if (pVCpu->hm.s.fSingleInstruction)
9774	{
9775	switch (uExitReason)
9776	{
9777	case VMX_EXIT_MTF:
9778	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9779
9780	/* Various events: */
9781	case VMX_EXIT_XCPT_OR_NMI:
9782	case VMX_EXIT_EXT_INT:
9783	case VMX_EXIT_TRIPLE_FAULT:
9784	case VMX_EXIT_INT_WINDOW:
9785	case VMX_EXIT_NMI_WINDOW:
9786	case VMX_EXIT_TASK_SWITCH:
9787	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9788	case VMX_EXIT_APIC_ACCESS:
9789	case VMX_EXIT_EPT_VIOLATION:
9790	case VMX_EXIT_EPT_MISCONFIG:
9791	case VMX_EXIT_PREEMPT_TIMER:
9792
9793	/* Instruction specific VM-exits: */
9794	case VMX_EXIT_CPUID:
9795	case VMX_EXIT_GETSEC:
9796	case VMX_EXIT_HLT:
9797	case VMX_EXIT_INVD:
9798	case VMX_EXIT_INVLPG:
9799	case VMX_EXIT_RDPMC:
9800	case VMX_EXIT_RDTSC:
9801	case VMX_EXIT_RSM:
9802	case VMX_EXIT_VMCALL:
9803	case VMX_EXIT_VMCLEAR:
9804	case VMX_EXIT_VMLAUNCH:
9805	case VMX_EXIT_VMPTRLD:
9806	case VMX_EXIT_VMPTRST:
9807	case VMX_EXIT_VMREAD:
9808	case VMX_EXIT_VMRESUME:
9809	case VMX_EXIT_VMWRITE:
9810	case VMX_EXIT_VMXOFF:
9811	case VMX_EXIT_VMXON:
9812	case VMX_EXIT_MOV_CRX:
9813	case VMX_EXIT_MOV_DRX:
9814	case VMX_EXIT_IO_INSTR:
9815	case VMX_EXIT_RDMSR:
9816	case VMX_EXIT_WRMSR:
9817	case VMX_EXIT_MWAIT:
9818	case VMX_EXIT_MONITOR:
9819	case VMX_EXIT_PAUSE:
9820	case VMX_EXIT_XDTR_ACCESS:
9821	case VMX_EXIT_TR_ACCESS:
9822	case VMX_EXIT_INVEPT:
9823	case VMX_EXIT_RDTSCP:
9824	case VMX_EXIT_INVVPID:
9825	case VMX_EXIT_WBINVD:
9826	case VMX_EXIT_XSETBV:
9827	case VMX_EXIT_RDRAND:
9828	case VMX_EXIT_INVPCID:
9829	case VMX_EXIT_VMFUNC:
9830	case VMX_EXIT_RDSEED:
9831	case VMX_EXIT_XSAVES:
9832	case VMX_EXIT_XRSTORS:
9833	{
9834	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9835	\| CPUMCTX_EXTRN_CS);
9836	AssertRCReturn(rc, rc);
9837	if ( pMixedCtx->rip != pDbgState->uRipStart
9838	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
9839	return VINF_EM_DBG_STEPPED;
9840	break;
9841	}
9842
9843	/* Errors and unexpected events: */
9844	case VMX_EXIT_INIT_SIGNAL:
9845	case VMX_EXIT_SIPI:
9846	case VMX_EXIT_IO_SMI:
9847	case VMX_EXIT_SMI:
9848	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9849	case VMX_EXIT_ERR_MSR_LOAD:
9850	case VMX_EXIT_ERR_MACHINE_CHECK:
9851	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9852	break;
9853
9854	default:
9855	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9856	break;
9857	}
9858	}
9859
9860	/*
9861	* Check for debugger event breakpoints and dtrace probes.
9862	*/
9863	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9864	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9865	{
9866	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9867	if (rcStrict != VINF_SUCCESS)
9868	return rcStrict;
9869	}
9870
9871	/*
9872	* Normal processing.
9873	*/
9874	#ifdef HMVMX_USE_FUNCTION_TABLE
9875	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9876	#else
9877	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9878	#endif
9879	}
9880
9881
9882	/**
9883	* Single steps guest code using VT-x.
9884	*
9885	* @returns Strict VBox status code (i.e. informational status codes too).
9886	* @param pVCpu The cross context virtual CPU structure.
9887	* @param pCtx Pointer to the guest-CPU context.
9888	*
9889	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9890	*/
9891	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9892	{
9893	VMXTRANSIENT VmxTransient;
9894	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9895
9896	/* Set HMCPU indicators. */
9897	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9898	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
9899	pVCpu->hm.s.fDebugWantRdTscExit = false;
9900	pVCpu->hm.s.fUsingDebugLoop = true;
9901
9902	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9903	VMXRUNDBGSTATE DbgState;
9904	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9905	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9906
9907	/*
9908	* The loop.
9909	*/
9910	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9911	for (uint32_t cLoops = 0; ; cLoops++)
9912	{
9913	Assert(!HMR0SuspendPending());
9914	HMVMX_ASSERT_CPU_SAFE();
9915	bool fStepping = pVCpu->hm.s.fSingleInstruction;
9916
9917	/*
9918	* Preparatory work for running guest code, this may force us to return
9919	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9920	*/
9921	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9922	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9923	rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9924	if (rcStrict != VINF_SUCCESS)
9925	break;
9926
9927	hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9928	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9929
9930	/*
9931	* Now we can run the guest code.
9932	*/
9933	int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9934
9935	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9936
9937	/*
9938	* Restore any residual host-state and save any bits shared between host
9939	* and guest into the guest-CPU state. Re-enables interrupts!
9940	*/
9941	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9942
9943	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9944	if (RT_SUCCESS(rcRun))
9945	{ /* very likely */ }
9946	else
9947	{
9948	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9949	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9950	return rcRun;
9951	}
9952
9953	/* Profile the VM-exit. */
9954	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9955	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9956	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9957	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9958	HMVMX_START_EXIT_DISPATCH_PROF();
9959
9960	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9961
9962	/*
9963	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9964	*/
9965	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9966	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9967	if (rcStrict != VINF_SUCCESS)
9968	break;
9969	if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9970	{
9971	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9972	rcStrict = VINF_EM_RAW_INTERRUPT;
9973	break;
9974	}
9975
9976	/*
9977	* Stepping: Did the RIP change, if so, consider it a single step.
9978	* Otherwise, make sure one of the TFs gets set.
9979	*/
9980	if (fStepping)
9981	{
9982	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9983	\| CPUMCTX_EXTRN_CS);
9984	AssertRC(rc);
9985	if ( pCtx->rip != DbgState.uRipStart
9986	\|\| pCtx->cs.Sel != DbgState.uCsStart)
9987	{
9988	rcStrict = VINF_EM_DBG_STEPPED;
9989	break;
9990	}
9991	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9992	}
9993
9994	/*
9995	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
9996	*/
9997	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9998	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9999	}
10000
10001	/*
10002	* Clear the X86_EFL_TF if necessary.
10003	*/
10004	if (pVCpu->hm.s.fClearTrapFlag)
10005	{
10006	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10007	AssertRC(rc);
10008	pVCpu->hm.s.fClearTrapFlag = false;
10009	pCtx->eflags.Bits.u1TF = 0;
10010	}
10011	/** @todo there seems to be issues with the resume flag when the monitor trap
10012	* flag is pending without being used. Seen early in bios init when
10013	* accessing APIC page in protected mode. */
10014
10015	/*
10016	* Restore VM-exit control settings as we may not reenter this function the
10017	* next time around.
10018	*/
10019	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10020
10021	/* Restore HMCPU indicators. */
10022	pVCpu->hm.s.fUsingDebugLoop = false;
10023	pVCpu->hm.s.fDebugWantRdTscExit = false;
10024	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10025
10026	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10027	return rcStrict;
10028	}
10029
10030
10031	/** @} */
10032
10033
10034	/**
10035	* Checks if any expensive dtrace probes are enabled and we should go to the
10036	* debug loop.
10037	*
10038	* @returns true if we should use debug loop, false if not.
10039	*/
10040	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10041	{
10042	/* It's probably faster to OR the raw 32-bit counter variables together.
10043	Since the variables are in an array and the probes are next to one
10044	another (more or less), we have good locality. So, better read
10045	eight-nine cache lines ever time and only have one conditional, than
10046	128+ conditionals, right? */
10047	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10048	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10049	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10050	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10051	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10052	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10053	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10054	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10055	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10056	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10057	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10058	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10059	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10060	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10061	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10062	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10063	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10064	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10065	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10066	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10067	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10068	) != 0
10069	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10070	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10071	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10072	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10073	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10074	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10075	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10076	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10077	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10078	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10079	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10080	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10081	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10082	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10083	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10084	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10085	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10086	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10087	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10088	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10089	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10090	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10091	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10092	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10093	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10094	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10095	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10096	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10097	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10098	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10099	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10100	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10101	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10102	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10103	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10104	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10105	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10106	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10107	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10108	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10109	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10110	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10111	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10112	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10113	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10114	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10115	) != 0
10116	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10117	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10118	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10119	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10120	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10121	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10122	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10123	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10124	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10125	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10126	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10127	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10128	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10129	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10130	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10131	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10132	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10133	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10134	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10135	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10136	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10137	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10138	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10139	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10140	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10141	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10142	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10143	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10144	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10145	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10146	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10147	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10148	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10149	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10150	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10151	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10152	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10153	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10154	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10155	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10156	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10157	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10158	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10159	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10160	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10161	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10162	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10163	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10164	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10165	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10166	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10167	) != 0;
10168	}
10169
10170
10171	/**
10172	* Runs the guest code using VT-x.
10173	*
10174	* @returns Strict VBox status code (i.e. informational status codes too).
10175	* @param pVCpu The cross context virtual CPU structure.
10176	* @param pCtx Pointer to the guest-CPU context.
10177	*/
10178	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10179	{
10180	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10181	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10182	HMVMX_ASSERT_PREEMPT_SAFE();
10183
10184	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10185
10186	VBOXSTRICTRC rcStrict;
10187	if ( !pVCpu->hm.s.fUseDebugLoop
10188	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10189	&& !DBGFIsStepping(pVCpu)
10190	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10191	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10192	else
10193	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10194
10195	if (rcStrict == VERR_EM_INTERPRETER)
10196	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10197	else if (rcStrict == VINF_EM_RESET)
10198	rcStrict = VINF_EM_TRIPLE_FAULT;
10199
10200	int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10201	if (RT_FAILURE(rc2))
10202	{
10203	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10204	rcStrict = rc2;
10205	}
10206	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10207	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10208	return rcStrict;
10209	}
10210
10211
10212	#ifndef HMVMX_USE_FUNCTION_TABLE
10213	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10214	{
10215	#ifdef DEBUG_ramshankar
10216	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10217	do { \
10218	if (a_fSave != 0) \
10219	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10220	VBOXSTRICTRC rcStrict = a_CallExpr; \
10221	if (a_fSave != 0) \
10222	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10223	return rcStrict; \
10224	} while (0)
10225	#else
10226	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10227	#endif
10228	switch (rcReason)
10229	{
10230	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10231	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10232	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10233	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10234	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10235	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10236	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10237	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10238	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10239	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10240	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10241	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10242	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10243	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10244	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10245	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10246	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10247	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10248	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10249	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10250	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10251	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10252	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10253	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10254	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10255	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10256	case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10257	case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10258	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10259	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10260	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10261	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10262	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10263	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10264
10265	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10266	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10267	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10268	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10269	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10270	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10271	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10272	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10273	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10274
10275	case VMX_EXIT_VMCLEAR:
10276	case VMX_EXIT_VMLAUNCH:
10277	case VMX_EXIT_VMPTRLD:
10278	case VMX_EXIT_VMPTRST:
10279	case VMX_EXIT_VMREAD:
10280	case VMX_EXIT_VMRESUME:
10281	case VMX_EXIT_VMWRITE:
10282	case VMX_EXIT_VMXOFF:
10283	case VMX_EXIT_VMXON:
10284	case VMX_EXIT_INVEPT:
10285	case VMX_EXIT_INVVPID:
10286	case VMX_EXIT_VMFUNC:
10287	case VMX_EXIT_XSAVES:
10288	case VMX_EXIT_XRSTORS:
10289	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10290
10291	case VMX_EXIT_ENCLS:
10292	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10293	case VMX_EXIT_PML_FULL:
10294	default:
10295	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10296	}
10297	#undef VMEXIT_CALL_RET
10298	}
10299	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10300
10301
10302	#ifdef VBOX_STRICT
10303	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10304	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10305	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10306
10307	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10308	do { \
10309	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10310	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10311	} while (0)
10312
10313	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10314	do { \
10315	AssertPtr(pVCpu); \
10316	AssertPtr(pMixedCtx); \
10317	AssertPtr(pVmxTransient); \
10318	Assert(pVmxTransient->fVMEntryFailed == false); \
10319	Assert(ASMIntAreEnabled()); \
10320	HMVMX_ASSERT_PREEMPT_SAFE(); \
10321	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10322	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10323	HMVMX_ASSERT_PREEMPT_SAFE(); \
10324	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10325	HMVMX_ASSERT_PREEMPT_CPUID(); \
10326	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10327	} while (0)
10328
10329	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10330	do { \
10331	Log4Func(("\n")); \
10332	} while (0)
10333	#else /* nonstrict builds: */
10334	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10335	do { \
10336	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10337	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10338	} while (0)
10339	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10340	#endif
10341
10342
10343	/**
10344	* Advances the guest RIP by the specified number of bytes.
10345	*
10346	* @param pVCpu The cross context virtual CPU structure.
10347	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10348	* out-of-sync. Make sure to update the required fields
10349	* before using them.
10350	* @param cbInstr Number of bytes to advance the RIP by.
10351	*
10352	* @remarks No-long-jump zone!!!
10353	*/
10354	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10355	{
10356	/* Advance the RIP. */
10357	pMixedCtx->rip += cbInstr;
10358	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10359
10360	/* Update interrupt inhibition. */
10361	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10362	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10363	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10364	}
10365
10366
10367	/**
10368	* Advances the guest RIP after reading it from the VMCS.
10369	*
10370	* @returns VBox status code, no informational status codes.
10371	* @param pVCpu The cross context virtual CPU structure.
10372	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10373	* out-of-sync. Make sure to update the required fields
10374	* before using them.
10375	* @param pVmxTransient Pointer to the VMX transient structure.
10376	*
10377	* @remarks No-long-jump zone!!!
10378	*/
10379	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10380	{
10381	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10382	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10383	\| CPUMCTX_EXTRN_RFLAGS);
10384	AssertRCReturn(rc, rc);
10385
10386	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10387
10388	/*
10389	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10390	* pending debug exception field as it takes care of priority of events.
10391	*
10392	* See Intel spec. 32.2.1 "Debug Exceptions".
10393	*/
10394	if ( !pVCpu->hm.s.fSingleInstruction
10395	&& pMixedCtx->eflags.Bits.u1TF)
10396	{
10397	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10398	AssertRCReturn(rc, rc);
10399	}
10400
10401	return VINF_SUCCESS;
10402	}
10403
10404
10405	/**
10406	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10407	* and update error record fields accordingly.
10408	*
10409	* @return VMX_IGS_* return codes.
10410	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10411	* wrong with the guest state.
10412	*
10413	* @param pVCpu The cross context virtual CPU structure.
10414	* @param pCtx Pointer to the guest-CPU state.
10415	*
10416	* @remarks This function assumes our cache of the VMCS controls
10417	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10418	*/
10419	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10420	{
10421	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10422	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10423	uError = (err); \
10424	break; \
10425	} else do { } while (0)
10426
10427	int rc;
10428	PVM pVM = pVCpu->CTX_SUFF(pVM);
10429	uint32_t uError = VMX_IGS_ERROR;
10430	uint32_t u32Val;
10431	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10432
10433	do
10434	{
10435	/*
10436	* CR0.
10437	*/
10438	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10439	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10440	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10441	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10442	if (fUnrestrictedGuest)
10443	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10444
10445	uint32_t u32GuestCr0;
10446	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10447	AssertRCBreak(rc);
10448	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10449	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10450	if ( !fUnrestrictedGuest
10451	&& (u32GuestCr0 & X86_CR0_PG)
10452	&& !(u32GuestCr0 & X86_CR0_PE))
10453	{
10454	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10455	}
10456
10457	/*
10458	* CR4.
10459	*/
10460	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10461	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10462
10463	uint32_t u32GuestCr4;
10464	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10465	AssertRCBreak(rc);
10466	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10467	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10468
10469	/*
10470	* IA32_DEBUGCTL MSR.
10471	*/
10472	uint64_t u64Val;
10473	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10474	AssertRCBreak(rc);
10475	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10476	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10477	{
10478	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10479	}
10480	uint64_t u64DebugCtlMsr = u64Val;
10481
10482	#ifdef VBOX_STRICT
10483	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10484	AssertRCBreak(rc);
10485	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10486	#endif
10487	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10488
10489	/*
10490	* RIP and RFLAGS.
10491	*/
10492	uint32_t u32Eflags;
10493	#if HC_ARCH_BITS == 64
10494	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10495	AssertRCBreak(rc);
10496	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10497	if ( !fLongModeGuest
10498	\|\| !pCtx->cs.Attr.n.u1Long)
10499	{
10500	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10501	}
10502	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10503	* must be identical if the "IA-32e mode guest" VM-entry
10504	* control is 1 and CS.L is 1. No check applies if the
10505	* CPU supports 64 linear-address bits. */
10506
10507	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10508	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10509	AssertRCBreak(rc);
10510	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10511	VMX_IGS_RFLAGS_RESERVED);
10512	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10513	u32Eflags = u64Val;
10514	#else
10515	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10516	AssertRCBreak(rc);
10517	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10518	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10519	#endif
10520
10521	if ( fLongModeGuest
10522	\|\| ( fUnrestrictedGuest
10523	&& !(u32GuestCr0 & X86_CR0_PE)))
10524	{
10525	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10526	}
10527
10528	uint32_t u32EntryInfo;
10529	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10530	AssertRCBreak(rc);
10531	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10532	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10533	{
10534	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10535	}
10536
10537	/*
10538	* 64-bit checks.
10539	*/
10540	#if HC_ARCH_BITS == 64
10541	if (fLongModeGuest)
10542	{
10543	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10544	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10545	}
10546
10547	if ( !fLongModeGuest
10548	&& (u32GuestCr4 & X86_CR4_PCIDE))
10549	{
10550	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10551	}
10552
10553	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10554	* 51:32 beyond the processor's physical-address width are 0. */
10555
10556	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10557	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10558	{
10559	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10560	}
10561
10562	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10563	AssertRCBreak(rc);
10564	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10565
10566	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10567	AssertRCBreak(rc);
10568	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10569	#endif
10570
10571	/*
10572	* PERF_GLOBAL MSR.
10573	*/
10574	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10575	{
10576	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10577	AssertRCBreak(rc);
10578	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10579	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10580	}
10581
10582	/*
10583	* PAT MSR.
10584	*/
10585	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10586	{
10587	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10588	AssertRCBreak(rc);
10589	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10590	for (unsigned i = 0; i < 8; i++)
10591	{
10592	uint8_t u8Val = (u64Val & 0xff);
10593	if ( u8Val != 0 /* UC */
10594	&& u8Val != 1 /* WC */
10595	&& u8Val != 4 /* WT */
10596	&& u8Val != 5 /* WP */
10597	&& u8Val != 6 /* WB */
10598	&& u8Val != 7 /* UC- */)
10599	{
10600	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10601	}
10602	u64Val >>= 8;
10603	}
10604	}
10605
10606	/*
10607	* EFER MSR.
10608	*/
10609	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10610	{
10611	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10612	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10613	AssertRCBreak(rc);
10614	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10615	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10616	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10617	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10618	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10619	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10620	\|\| !(u32GuestCr0 & X86_CR0_PG)
10621	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10622	VMX_IGS_EFER_LMA_LME_MISMATCH);
10623	}
10624
10625	/*
10626	* Segment registers.
10627	*/
10628	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10629	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10630	if (!(u32Eflags & X86_EFL_VM))
10631	{
10632	/* CS */
10633	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10634	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10635	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10636	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10637	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10638	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10639	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10640	/* CS cannot be loaded with NULL in protected mode. */
10641	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10642	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10643	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10644	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10645	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10646	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10647	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10648	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10649	else
10650	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10651
10652	/* SS */
10653	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10654	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10655	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10656	if ( !(pCtx->cr0 & X86_CR0_PE)
10657	\|\| pCtx->cs.Attr.n.u4Type == 3)
10658	{
10659	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10660	}
10661	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10662	{
10663	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10664	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10665	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10666	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10667	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10668	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10669	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10670	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10671	}
10672
10673	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10674	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10675	{
10676	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10677	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10678	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10679	\|\| pCtx->ds.Attr.n.u4Type > 11
10680	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10681	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10682	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10683	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10684	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10685	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10686	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10687	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10688	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10689	}
10690	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10691	{
10692	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10693	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10694	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10695	\|\| pCtx->es.Attr.n.u4Type > 11
10696	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10697	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10698	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10699	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10700	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10701	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10702	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10703	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10704	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10705	}
10706	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10707	{
10708	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10709	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10710	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10711	\|\| pCtx->fs.Attr.n.u4Type > 11
10712	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10713	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10714	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10715	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10716	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10717	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10718	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10719	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10720	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10721	}
10722	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10723	{
10724	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10725	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10726	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10727	\|\| pCtx->gs.Attr.n.u4Type > 11
10728	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10729	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10730	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10731	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10732	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10733	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10734	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10735	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10736	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10737	}
10738	/* 64-bit capable CPUs. */
10739	#if HC_ARCH_BITS == 64
10740	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10741	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10742	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10743	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10744	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10745	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10746	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10747	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10748	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10749	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10750	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10751	#endif
10752	}
10753	else
10754	{
10755	/* V86 mode checks. */
10756	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10757	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10758	{
10759	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10760	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10761	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10762	}
10763	else
10764	{
10765	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10766	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10767	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10768	}
10769
10770	/* CS */
10771	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10772	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10773	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10774	/* SS */
10775	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10776	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10777	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10778	/* DS */
10779	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10780	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10781	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10782	/* ES */
10783	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10784	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10785	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10786	/* FS */
10787	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10788	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10789	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10790	/* GS */
10791	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10792	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10793	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10794	/* 64-bit capable CPUs. */
10795	#if HC_ARCH_BITS == 64
10796	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10797	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10798	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10799	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10800	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10801	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10802	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10803	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10804	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10805	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10806	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10807	#endif
10808	}
10809
10810	/*
10811	* TR.
10812	*/
10813	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10814	/* 64-bit capable CPUs. */
10815	#if HC_ARCH_BITS == 64
10816	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10817	#endif
10818	if (fLongModeGuest)
10819	{
10820	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10821	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10822	}
10823	else
10824	{
10825	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10826	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10827	VMX_IGS_TR_ATTR_TYPE_INVALID);
10828	}
10829	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10830	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10831	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10832	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10833	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10834	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10835	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10836	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10837
10838	/*
10839	* GDTR and IDTR.
10840	*/
10841	#if HC_ARCH_BITS == 64
10842	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10843	AssertRCBreak(rc);
10844	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10845
10846	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10847	AssertRCBreak(rc);
10848	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10849	#endif
10850
10851	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10852	AssertRCBreak(rc);
10853	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10854
10855	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10856	AssertRCBreak(rc);
10857	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10858
10859	/*
10860	* Guest Non-Register State.
10861	*/
10862	/* Activity State. */
10863	uint32_t u32ActivityState;
10864	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10865	AssertRCBreak(rc);
10866	HMVMX_CHECK_BREAK( !u32ActivityState
10867	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10868	VMX_IGS_ACTIVITY_STATE_INVALID);
10869	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10870	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10871	uint32_t u32IntrState;
10872	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10873	AssertRCBreak(rc);
10874	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10875	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10876	{
10877	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10878	}
10879
10880	/** @todo Activity state and injecting interrupts. Left as a todo since we
10881	* currently don't use activity states but ACTIVE. */
10882
10883	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10884	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10885
10886	/* Guest interruptibility-state. */
10887	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10888	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10889	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10890	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10891	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10892	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10893	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10894	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10895	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10896	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10897	{
10898	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10899	{
10900	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10901	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10902	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10903	}
10904	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10905	{
10906	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10907	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10908	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10909	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10910	}
10911	}
10912	/** @todo Assumes the processor is not in SMM. */
10913	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10914	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10915	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10916	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10917	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10918	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10919	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10920	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10921	{
10922	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10923	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10924	}
10925
10926	/* Pending debug exceptions. */
10927	#if HC_ARCH_BITS == 64
10928	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10929	AssertRCBreak(rc);
10930	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10931	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10932	u32Val = u64Val; /* For pending debug exceptions checks below. */
10933	#else
10934	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10935	AssertRCBreak(rc);
10936	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10937	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10938	#endif
10939
10940	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10941	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10942	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10943	{
10944	if ( (u32Eflags & X86_EFL_TF)
10945	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10946	{
10947	/* Bit 14 is PendingDebug.BS. */
10948	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10949	}
10950	if ( !(u32Eflags & X86_EFL_TF)
10951	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10952	{
10953	/* Bit 14 is PendingDebug.BS. */
10954	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10955	}
10956	}
10957
10958	/* VMCS link pointer. */
10959	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10960	AssertRCBreak(rc);
10961	if (u64Val != UINT64_C(0xffffffffffffffff))
10962	{
10963	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10964	/** @todo Bits beyond the processor's physical-address width MBZ. */
10965	/** @todo 32-bit located in memory referenced by value of this field (as a
10966	* physical address) must contain the processor's VMCS revision ID. */
10967	/** @todo SMM checks. */
10968	}
10969
10970	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10971	* not using Nested Paging? */
10972	if ( pVM->hm.s.fNestedPaging
10973	&& !fLongModeGuest
10974	&& CPUMIsGuestInPAEModeEx(pCtx))
10975	{
10976	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10977	AssertRCBreak(rc);
10978	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10979
10980	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10981	AssertRCBreak(rc);
10982	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10983
10984	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10985	AssertRCBreak(rc);
10986	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10987
10988	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10989	AssertRCBreak(rc);
10990	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10991	}
10992
10993	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10994	if (uError == VMX_IGS_ERROR)
10995	uError = VMX_IGS_REASON_NOT_FOUND;
10996	} while (0);
10997
10998	pVCpu->hm.s.u32HMError = uError;
10999	return uError;
11000
11001	#undef HMVMX_ERROR_BREAK
11002	#undef HMVMX_CHECK_BREAK
11003	}
11004
11005	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11006	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11007	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11008
11009	/** @name VM-exit handlers.
11010	* @{
11011	*/
11012
11013	/**
11014	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11015	*/
11016	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11017	{
11018	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11019	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11020	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11021	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11022	return VINF_SUCCESS;
11023	return VINF_EM_RAW_INTERRUPT;
11024	}
11025
11026
11027	/**
11028	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11029	*/
11030	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11031	{
11032	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11033	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11034
11035	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11036	AssertRCReturn(rc, rc);
11037
11038	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11039	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11040	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11041	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11042
11043	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11044	{
11045	/*
11046	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11047	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
11048	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11049	*
11050	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11051	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11052	*/
11053	VMXDispatchHostNmi();
11054	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11055	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11056	return VINF_SUCCESS;
11057	}
11058
11059	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11060	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11061	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11062	{ /* likely */ }
11063	else
11064	{
11065	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11066	rcStrictRc1 = VINF_SUCCESS;
11067	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11068	return rcStrictRc1;
11069	}
11070
11071	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11072	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11073	switch (uIntType)
11074	{
11075	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11076	Assert(uVector == X86_XCPT_DB);
11077	RT_FALL_THRU();
11078	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11079	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11080	RT_FALL_THRU();
11081	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11082	{
11083	/*
11084	* If there's any exception caused as a result of event injection, the resulting
11085	* secondary/final execption will be pending, we shall continue guest execution
11086	* after injecting the event. The page-fault case is complicated and we manually
11087	* handle any currently pending event in hmR0VmxExitXcptPF.
11088	*/
11089	if (!pVCpu->hm.s.Event.fPending)
11090	{ /* likely */ }
11091	else if (uVector != X86_XCPT_PF)
11092	{
11093	rc = VINF_SUCCESS;
11094	break;
11095	}
11096
11097	switch (uVector)
11098	{
11099	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11100	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11101	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11102	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11103	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11104	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11105
11106	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11107	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11108	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11109	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11110	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11111	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11112	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11113	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11114	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11115	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11116	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11117	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11118	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11119	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11120	default:
11121	{
11122	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11123	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11124	{
11125	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11126	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11127	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11128
11129	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11130	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11131	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11132	AssertRCReturn(rc, rc);
11133	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11134	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11135	0 /* GCPtrFaultAddress */);
11136	}
11137	else
11138	{
11139	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11140	pVCpu->hm.s.u32HMError = uVector;
11141	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11142	}
11143	break;
11144	}
11145	}
11146	break;
11147	}
11148
11149	default:
11150	{
11151	pVCpu->hm.s.u32HMError = uExitIntInfo;
11152	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11153	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11154	break;
11155	}
11156	}
11157	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11158	return rc;
11159	}
11160
11161
11162	/**
11163	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11164	*/
11165	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11166	{
11167	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11168
11169	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11170	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11171
11172	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11173	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11174	return VINF_SUCCESS;
11175	}
11176
11177
11178	/**
11179	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11180	*/
11181	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11182	{
11183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11184	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11185	{
11186	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11187	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11188	}
11189
11190	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11191
11192	/*
11193	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11194	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11195	*/
11196	uint32_t fIntrState = 0;
11197	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11198	AssertRCReturn(rc, rc);
11199
11200	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11201	if ( fBlockSti
11202	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11203	{
11204	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11205	}
11206
11207	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11208	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11209
11210	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11211	return VINF_SUCCESS;
11212	}
11213
11214
11215	/**
11216	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11217	*/
11218	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11219	{
11220	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11221	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11222	}
11223
11224
11225	/**
11226	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11227	*/
11228	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11229	{
11230	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11231	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11232	}
11233
11234
11235	/**
11236	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11237	*/
11238	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11239	{
11240	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11241	Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11242
11243	/*
11244	* Get the state we need and update the exit history entry.
11245	*/
11246	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11247	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11248	\| CPUMCTX_EXTRN_CS);
11249	AssertRCReturn(rc, rc);
11250
11251	VBOXSTRICTRC rcStrict;
11252	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11253	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
11254	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11255	if (!pExitRec)
11256	{
11257	/*
11258	* Regular CPUID instruction execution.
11259	*/
11260	PVM pVM = pVCpu->CTX_SUFF(pVM);
11261	rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11262	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11263	{
11264	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11265	Assert(pVmxTransient->cbInstr == 2);
11266	}
11267	else
11268	{
11269	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11270	rcStrict = VERR_EM_INTERPRETER;
11271	}
11272	}
11273	else
11274	{
11275	/*
11276	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
11277	*/
11278	Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11279	int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11280	AssertRCReturn(rc2, rc2);
11281
11282	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11283	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11284
11285	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11286	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11287
11288	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11289	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11290	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11291	}
11292	return VBOXSTRICTRC_TODO(rcStrict);
11293	}
11294
11295
11296	/**
11297	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11298	*/
11299	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11300	{
11301	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11302	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11303	AssertRCReturn(rc, rc);
11304
11305	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11306	return VINF_EM_RAW_EMULATE_INSTR;
11307
11308	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11309	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11310	}
11311
11312
11313	/**
11314	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11315	*/
11316	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11317	{
11318	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11319	int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11320	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11321	AssertRCReturn(rc, rc);
11322
11323	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11324	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11325	{
11326	/* If we get a spurious VM-exit when offsetting is enabled,
11327	we must reset offsetting on VM-reentry. See @bugref{6634}. */
11328	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11329	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11330	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11331	\| HM_CHANGED_GUEST_RFLAGS);
11332	}
11333	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11334	{
11335	rcStrict = VINF_SUCCESS;
11336	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11337	}
11338	return rcStrict;
11339	}
11340
11341
11342	/**
11343	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11344	*/
11345	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11346	{
11347	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11348	int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11349	\| CPUMCTX_EXTRN_TSC_AUX);
11350	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11351	AssertRCReturn(rc, rc);
11352
11353	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11354	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11355	{
11356	/* If we get a spurious VM-exit when offsetting is enabled,
11357	we must reset offsetting on VM-reentry. See @bugref{6634}. */
11358	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11359	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11360	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11361	\| HM_CHANGED_GUEST_RFLAGS);
11362	}
11363	else if (rcStrict == VINF_IEM_RAISED_XCPT)
11364	{
11365	rcStrict = VINF_SUCCESS;
11366	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11367	}
11368	return rcStrict;
11369	}
11370
11371
11372	/**
11373	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11374	*/
11375	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11376	{
11377	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11378	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11379	\| CPUMCTX_EXTRN_CR0
11380	\| CPUMCTX_EXTRN_RFLAGS
11381	\| CPUMCTX_EXTRN_SS);
11382	AssertRCReturn(rc, rc);
11383
11384	PVM pVM = pVCpu->CTX_SUFF(pVM);
11385	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11386	if (RT_LIKELY(rc == VINF_SUCCESS))
11387	{
11388	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11389	Assert(pVmxTransient->cbInstr == 2);
11390	}
11391	else
11392	{
11393	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11394	rc = VERR_EM_INTERPRETER;
11395	}
11396	return rc;
11397	}
11398
11399
11400	/**
11401	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11402	*/
11403	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11404	{
11405	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11406
11407	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11408	if (EMAreHypercallInstructionsEnabled(pVCpu))
11409	{
11410	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11411	\| CPUMCTX_EXTRN_RFLAGS
11412	\| CPUMCTX_EXTRN_CR0
11413	\| CPUMCTX_EXTRN_SS
11414	\| CPUMCTX_EXTRN_CS
11415	\| CPUMCTX_EXTRN_EFER);
11416	AssertRCReturn(rc, rc);
11417
11418	/* Perform the hypercall. */
11419	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11420	if (rcStrict == VINF_SUCCESS)
11421	{
11422	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11423	AssertRCReturn(rc, rc);
11424	}
11425	else
11426	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11427	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11428	\|\| RT_FAILURE(rcStrict));
11429
11430	/* If the hypercall changes anything other than guest's general-purpose registers,
11431	we would need to reload the guest changed bits here before VM-entry. */
11432	}
11433	else
11434	Log4Func(("Hypercalls not enabled\n"));
11435
11436	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11437	if (RT_FAILURE(rcStrict))
11438	{
11439	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11440	rcStrict = VINF_SUCCESS;
11441	}
11442
11443	return rcStrict;
11444	}
11445
11446
11447	/**
11448	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11449	*/
11450	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11451	{
11452	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11453	PVM pVM = pVCpu->CTX_SUFF(pVM);
11454	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11455
11456	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11457	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11458	AssertRCReturn(rc, rc);
11459
11460	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11461	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11462	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11463	else
11464	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11465	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11466	return rcStrict;
11467	}
11468
11469
11470	/**
11471	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11472	*/
11473	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11474	{
11475	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11476	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11477	\| CPUMCTX_EXTRN_RFLAGS
11478	\| CPUMCTX_EXTRN_SS);
11479	AssertRCReturn(rc, rc);
11480
11481	PVM pVM = pVCpu->CTX_SUFF(pVM);
11482	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11483	if (RT_LIKELY(rc == VINF_SUCCESS))
11484	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11485	else
11486	{
11487	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11488	rc = VERR_EM_INTERPRETER;
11489	}
11490	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11491	return rc;
11492	}
11493
11494
11495	/**
11496	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11497	*/
11498	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11499	{
11500	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11501	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11502	\| CPUMCTX_EXTRN_RFLAGS
11503	\| CPUMCTX_EXTRN_SS);
11504	AssertRCReturn(rc, rc);
11505
11506	PVM pVM = pVCpu->CTX_SUFF(pVM);
11507	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11508	rc = VBOXSTRICTRC_VAL(rc2);
11509	if (RT_LIKELY( rc == VINF_SUCCESS
11510	\|\| rc == VINF_EM_HALT))
11511	{
11512	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11513	AssertRCReturn(rc3, rc3);
11514
11515	if ( rc == VINF_EM_HALT
11516	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11517	rc = VINF_SUCCESS;
11518	}
11519	else
11520	{
11521	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11522	rc = VERR_EM_INTERPRETER;
11523	}
11524	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11525	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11526	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11527	return rc;
11528	}
11529
11530
11531	/**
11532	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11533	*/
11534	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11535	{
11536	/*
11537	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11538	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11539	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11540	* VMX root operation. If we get here, something funny is going on.
11541	*
11542	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11543	*/
11544	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11545	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11546	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11547	}
11548
11549
11550	/**
11551	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11552	*/
11553	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11554	{
11555	/*
11556	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
11557	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11558	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11559	* an SMI. If we get here, something funny is going on.
11560	*
11561	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11562	* See Intel spec. 25.3 "Other Causes of VM-Exits"
11563	*/
11564	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11565	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11566	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11567	}
11568
11569
11570	/**
11571	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11572	*/
11573	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11574	{
11575	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11576	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11577	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11578	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11579	}
11580
11581
11582	/**
11583	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11584	*/
11585	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11586	{
11587	/*
11588	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11589	* We don't make use of it as our guests don't have direct access to the host LAPIC.
11590	* See Intel spec. 25.3 "Other Causes of VM-exits".
11591	*/
11592	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11593	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11594	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11595	}
11596
11597
11598	/**
11599	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11600	* VM-exit.
11601	*/
11602	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11603	{
11604	/*
11605	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11606	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11607	*
11608	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11609	* See Intel spec. "23.8 Restrictions on VMX operation".
11610	*/
11611	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11612	return VINF_SUCCESS;
11613	}
11614
11615
11616	/**
11617	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11618	* VM-exit.
11619	*/
11620	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11621	{
11622	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11623	return VINF_EM_RESET;
11624	}
11625
11626
11627	/**
11628	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11629	*/
11630	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11631	{
11632	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11633	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11634
11635	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11636	AssertRCReturn(rc, rc);
11637
11638	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11639	rc = VINF_SUCCESS;
11640	else
11641	rc = VINF_EM_HALT;
11642
11643	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11644	if (rc != VINF_SUCCESS)
11645	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11646	return rc;
11647	}
11648
11649
11650	/**
11651	* VM-exit handler for instructions that result in a \#UD exception delivered to
11652	* the guest.
11653	*/
11654	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11655	{
11656	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11657	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11658	return VINF_SUCCESS;
11659	}
11660
11661
11662	/**
11663	* VM-exit handler for expiry of the VMX preemption timer.
11664	*/
11665	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11666	{
11667	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11668
11669	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11670	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11671
11672	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11673	PVM pVM = pVCpu->CTX_SUFF(pVM);
11674	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11675	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11676	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11677	}
11678
11679
11680	/**
11681	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11682	*/
11683	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11684	{
11685	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11686
11687	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11688	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11689	\| CPUMCTX_EXTRN_CR4);
11690	AssertRCReturn(rc, rc);
11691
11692	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11693	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
11694	: HM_CHANGED_XCPT_RAISED_MASK);
11695
11696	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11697
11698	return rcStrict;
11699	}
11700
11701
11702	/**
11703	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11704	*/
11705	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11706	{
11707	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11708	/** @todo Use VM-exit instruction information. */
11709	return VERR_EM_INTERPRETER;
11710	}
11711
11712
11713	/**
11714	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11715	* Error VM-exit.
11716	*/
11717	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11718	{
11719	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11720	AssertRCReturn(rc, rc);
11721	rc = hmR0VmxCheckVmcsCtls(pVCpu);
11722	if (RT_FAILURE(rc))
11723	return rc;
11724
11725	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11726	NOREF(uInvalidReason);
11727
11728	#ifdef VBOX_STRICT
11729	uint32_t fIntrState;
11730	RTHCUINTREG uHCReg;
11731	uint64_t u64Val;
11732	uint32_t u32Val;
11733
11734	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11735	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11736	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11737	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11738	AssertRCReturn(rc, rc);
11739
11740	Log4(("uInvalidReason %u\n", uInvalidReason));
11741	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11742	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11743	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11744	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11745
11746	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11747	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11748	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11749	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11750	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11751	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11752	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11753	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11754	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11755	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11756	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11757	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11758
11759	hmR0DumpRegs(pVCpu, pMixedCtx);
11760	#else
11761	NOREF(pVmxTransient);
11762	#endif
11763
11764	return VERR_VMX_INVALID_GUEST_STATE;
11765	}
11766
11767
11768	/**
11769	* VM-exit handler for VM-entry failure due to an MSR-load
11770	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11771	*/
11772	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11773	{
11774	NOREF(pVmxTransient);
11775	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11776	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11777	}
11778
11779
11780	/**
11781	* VM-exit handler for VM-entry failure due to a machine-check event
11782	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11783	*/
11784	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11785	{
11786	NOREF(pVmxTransient);
11787	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11788	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11789	}
11790
11791
11792	/**
11793	* VM-exit handler for all undefined reasons. Should never ever happen.. in
11794	* theory.
11795	*/
11796	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11797	{
11798	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11799	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11800	return VERR_VMX_UNDEFINED_EXIT_CODE;
11801	}
11802
11803
11804	/**
11805	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11806	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11807	* Conditional VM-exit.
11808	*/
11809	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11810	{
11811	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11812
11813	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11814	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11815	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11816	return VERR_EM_INTERPRETER;
11817	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11818	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11819	}
11820
11821
11822	/**
11823	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11824	*/
11825	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11826	{
11827	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11828
11829	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11830	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11831	return VERR_EM_INTERPRETER;
11832	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11833	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11834	}
11835
11836
11837	/**
11838	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11839	*/
11840	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11841	{
11842	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11843
11844	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. FS, GS (base) can be accessed by MSR reads. */
11845	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11846	\| CPUMCTX_EXTRN_RFLAGS
11847	\| CPUMCTX_EXTRN_SS
11848	\| CPUMCTX_EXTRN_FS
11849	\| CPUMCTX_EXTRN_GS);
11850	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11851	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11852	AssertRCReturn(rc, rc);
11853	Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11854
11855	#ifdef VBOX_STRICT
11856	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11857	{
11858	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11859	&& pMixedCtx->ecx != MSR_K6_EFER)
11860	{
11861	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11862	pMixedCtx->ecx));
11863	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11864	}
11865	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11866	{
11867	VMXMSREXITREAD enmRead;
11868	VMXMSREXITWRITE enmWrite;
11869	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11870	AssertRCReturn(rc2, rc2);
11871	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11872	{
11873	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11874	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11875	}
11876	}
11877	}
11878	#endif
11879
11880	PVM pVM = pVCpu->CTX_SUFF(pVM);
11881	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11882	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
11883	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11885	if (RT_SUCCESS(rc))
11886	{
11887	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11888	Assert(pVmxTransient->cbInstr == 2);
11889	}
11890	return rc;
11891	}
11892
11893
11894	/**
11895	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11896	*/
11897	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11898	{
11899	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11900	PVM pVM = pVCpu->CTX_SUFF(pVM);
11901	int rc = VINF_SUCCESS;
11902
11903	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. FS, GS (base) can be accessed by MSR writes. */
11904	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11905	\| CPUMCTX_EXTRN_RFLAGS
11906	\| CPUMCTX_EXTRN_SS
11907	\| CPUMCTX_EXTRN_FS
11908	\| CPUMCTX_EXTRN_GS);
11909	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11910	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11911	AssertRCReturn(rc, rc);
11912	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11913
11914	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11915	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11916	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11917
11918	if (RT_SUCCESS(rc))
11919	{
11920	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11921
11922	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
11923	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11924	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11925	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11926	{
11927	/*
11928	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11929	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11930	* EMInterpretWrmsr() changes it.
11931	*/
11932	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11933	}
11934	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11935	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11936	else if (pMixedCtx->ecx == MSR_K6_EFER)
11937	{
11938	/*
11939	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11940	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11941	* the other bits as well, SCE and NXE. See @bugref{7368}.
11942	*/
11943	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11944	\| HM_CHANGED_VMX_ENTRY_CTLS
11945	\| HM_CHANGED_VMX_EXIT_CTLS);
11946	}
11947
11948	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11949	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11950	{
11951	switch (pMixedCtx->ecx)
11952	{
11953	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11954	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11955	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11956	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11957	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11958	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11959	default:
11960	{
11961	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11962	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11963	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11964	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11965	break;
11966	}
11967	}
11968	}
11969	#ifdef VBOX_STRICT
11970	else
11971	{
11972	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11973	switch (pMixedCtx->ecx)
11974	{
11975	case MSR_IA32_SYSENTER_CS:
11976	case MSR_IA32_SYSENTER_EIP:
11977	case MSR_IA32_SYSENTER_ESP:
11978	case MSR_K8_FS_BASE:
11979	case MSR_K8_GS_BASE:
11980	{
11981	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11982	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11983	}
11984
11985	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11986	default:
11987	{
11988	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11989	{
11990	/* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11991	if (pMixedCtx->ecx != MSR_K6_EFER)
11992	{
11993	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11994	pMixedCtx->ecx));
11995	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11996	}
11997	}
11998
11999	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12000	{
12001	VMXMSREXITREAD enmRead;
12002	VMXMSREXITWRITE enmWrite;
12003	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12004	AssertRCReturn(rc2, rc2);
12005	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12006	{
12007	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12008	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12009	}
12010	}
12011	break;
12012	}
12013	}
12014	}
12015	#endif /* VBOX_STRICT */
12016	}
12017	return rc;
12018	}
12019
12020
12021	/**
12022	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12023	*/
12024	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12025	{
12026	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12027	/** @todo The guest has likely hit a contended spinlock. We might want to
12028	* poke a schedule different guest VCPU. */
12029	return VINF_EM_RAW_INTERRUPT;
12030	}
12031
12032
12033	/**
12034	* VM-exit handler for when the TPR value is lowered below the specified
12035	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12036	*/
12037	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12038	{
12039	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12040	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12041
12042	/*
12043	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12044	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12045	*/
12046	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12047	return VINF_SUCCESS;
12048	}
12049
12050
12051	/**
12052	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12053	* VM-exit.
12054	*
12055	* @retval VINF_SUCCESS when guest execution can continue.
12056	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12057	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12058	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12059	* interpreter.
12060	*/
12061	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12062	{
12063	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12064	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12065
12066	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12067	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12068	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12069	AssertRCReturn(rc, rc);
12070
12071	VBOXSTRICTRC rcStrict;
12072	PVM pVM = pVCpu->CTX_SUFF(pVM);
12073	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12074	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
12075	switch (uAccessType)
12076	{
12077	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12078	{
12079	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12080	VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12081	VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
12082	AssertMsg( rcStrict == VINF_SUCCESS
12083	\|\| rcStrict == VINF_IEM_RAISED_XCPT
12084	\|\| rcStrict == VINF_PGM_CHANGE_MODE
12085	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12086
12087	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12088	{
12089	case 0:
12090	{
12091	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12093	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12094	break;
12095	}
12096
12097	case 2:
12098	{
12099	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12100	/* Nothing to do here, CR2 it's not part of the VMCS. */
12101	break;
12102	}
12103
12104	case 3:
12105	{
12106	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12107	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12108	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
12109	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12110	break;
12111	}
12112
12113	case 4:
12114	{
12115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12116	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12117	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12118	pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12119	break;
12120	}
12121
12122	case 8:
12123	{
12124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12125	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12126	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12127	break;
12128	}
12129	default:
12130	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12131	break;
12132	}
12133	break;
12134	}
12135
12136	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12137	{
12138	Assert( !pVM->hm.s.fNestedPaging
12139	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12140	\|\| pVCpu->hm.s.fUsingDebugLoop
12141	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12142	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12143	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12144	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12145
12146	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12147	VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12148	VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12149	AssertMsg( rcStrict == VINF_SUCCESS
12150	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12151	#ifdef VBOX_WITH_STATISTICS
12152	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12153	{
12154	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12155	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12156	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12157	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12158	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12159	}
12160	#endif
12161	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12162	VBOXSTRICTRC_VAL(rcStrict)));
12163	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12164	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12165	break;
12166	}
12167
12168	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12169	{
12170	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12171	AssertMsg( rcStrict == VINF_SUCCESS
12172	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12173
12174	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12175	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12176	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12177	break;
12178	}
12179
12180	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12181	{
12182	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12183	VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12184	AssertMsg( rcStrict == VINF_SUCCESS
12185	\|\| rcStrict == VINF_IEM_RAISED_XCPT
12186	\|\| rcStrict == VINF_PGM_CHANGE_MODE,
12187	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12188
12189	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12190	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12191	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12192	break;
12193	}
12194
12195	default:
12196	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12197	VERR_VMX_UNEXPECTED_EXCEPTION);
12198	}
12199
12200	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
12201	: HM_CHANGED_XCPT_RAISED_MASK);
12202	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12203	NOREF(pVM);
12204	return rcStrict;
12205	}
12206
12207
12208	/**
12209	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12210	* VM-exit.
12211	*/
12212	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12213	{
12214	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12215	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12216	Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12217
12218	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12219	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12220	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12221	\| CPUMCTX_EXTRN_SREG_MASK
12222	\| CPUMCTX_EXTRN_EFER);
12223	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12224	AssertRCReturn(rc, rc);
12225
12226	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12227	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12228	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12229	bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12230	== VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12231	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12232	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12233	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12234	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12235
12236	/*
12237	* Update exit history to see if this exit can be optimized.
12238	*/
12239	VBOXSTRICTRC rcStrict;
12240	PCEMEXITREC pExitRec = NULL;
12241	if ( !fGstStepping
12242	&& !fDbgStepping)
12243	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12244	!fIOString
12245	? !fIOWrite
12246	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12247	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12248	: !fIOWrite
12249	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12250	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12251	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12252	if (!pExitRec)
12253	{
12254	/* I/O operation lookup arrays. */
12255	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12256	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12257	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12258	uint32_t const cbInstr = pVmxTransient->cbInstr;
12259	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12260	PVM pVM = pVCpu->CTX_SUFF(pVM);
12261	if (fIOString)
12262	{
12263	/*
12264	* INS/OUTS - I/O String instruction.
12265	*
12266	* Use instruction-information if available, otherwise fall back on
12267	* interpreting the instruction.
12268	*/
12269	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12270	fIOWrite ? 'w' : 'r'));
12271	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12272	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12273	{
12274	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12275	AssertRCReturn(rc2, rc2);
12276	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12277	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12278	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12279	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12280	if (fIOWrite)
12281	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12282	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12283	else
12284	{
12285	/*
12286	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12287	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12288	* See Intel Instruction spec. for "INS".
12289	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12290	*/
12291	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12292	}
12293	}
12294	else
12295	rcStrict = IEMExecOne(pVCpu);
12296
12297	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12298	fUpdateRipAlready = true;
12299	}
12300	else
12301	{
12302	/*
12303	* IN/OUT - I/O instruction.
12304	*/
12305	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12306	fIOWrite ? 'w' : 'r'));
12307	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12308	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12309	if (fIOWrite)
12310	{
12311	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12312	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12313	}
12314	else
12315	{
12316	uint32_t u32Result = 0;
12317	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12318	if (IOM_SUCCESS(rcStrict))
12319	{
12320	/* Save result of I/O IN instr. in AL/AX/EAX. */
12321	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12322	}
12323	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12324	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12325	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12326	}
12327	}
12328
12329	if (IOM_SUCCESS(rcStrict))
12330	{
12331	if (!fUpdateRipAlready)
12332	{
12333	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12334	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12335	}
12336
12337	/*
12338	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12339	* while booting Fedora 17 64-bit guest.
12340	*
12341	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12342	*/
12343	if (fIOString)
12344	{
12345	/** @todo Single-step for INS/OUTS with REP prefix? */
12346	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12347	}
12348	else if ( !fDbgStepping
12349	&& fGstStepping)
12350	{
12351	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12352	AssertRCReturn(rc, rc);
12353	}
12354
12355	/*
12356	* If any I/O breakpoints are armed, we need to check if one triggered
12357	* and take appropriate action.
12358	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12359	*/
12360	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12361	AssertRCReturn(rc, rc);
12362
12363	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12364	* execution engines about whether hyper BPs and such are pending. */
12365	uint32_t const uDr7 = pMixedCtx->dr[7];
12366	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12367	&& X86_DR7_ANY_RW_IO(uDr7)
12368	&& (pMixedCtx->cr4 & X86_CR4_DE))
12369	\|\| DBGFBpIsHwIoArmed(pVM)))
12370	{
12371	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12372
12373	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12374	VMMRZCallRing3Disable(pVCpu);
12375	HM_DISABLE_PREEMPT();
12376
12377	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12378
12379	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12380	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12381	{
12382	/* Raise #DB. */
12383	if (fIsGuestDbgActive)
12384	ASMSetDR6(pMixedCtx->dr[6]);
12385	if (pMixedCtx->dr[7] != uDr7)
12386	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
12387
12388	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12389	}
12390	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12391	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12392	else if ( rcStrict2 != VINF_SUCCESS
12393	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12394	rcStrict = rcStrict2;
12395	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12396
12397	HM_RESTORE_PREEMPT();
12398	VMMRZCallRing3Enable(pVCpu);
12399	}
12400	}
12401
12402	#ifdef VBOX_STRICT
12403	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12404	Assert(!fIOWrite);
12405	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12406	Assert(fIOWrite);
12407	else
12408	{
12409	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12410	* statuses, that the VMM device and some others may return. See
12411	* IOM_SUCCESS() for guidance. */
12412	AssertMsg( RT_FAILURE(rcStrict)
12413	\|\| rcStrict == VINF_SUCCESS
12414	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12415	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12416	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12417	\|\| rcStrict == VINF_EM_RAW_TO_R3
12418	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12419	# endif
12420	}
12421	#endif
12422	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12423	}
12424	else
12425	{
12426	/*
12427	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
12428	*/
12429	int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12430	AssertRCReturn(rc2, rc2);
12431	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12432	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12433	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12434	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12435	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12436	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12437
12438	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12439	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12440
12441	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12442	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12443	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12444	}
12445	return rcStrict;
12446	}
12447
12448
12449	/**
12450	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12451	* VM-exit.
12452	*/
12453	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12454	{
12455	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12456
12457	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12458	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12459	AssertRCReturn(rc, rc);
12460	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12461	{
12462	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12463	AssertRCReturn(rc, rc);
12464	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12465	{
12466	uint32_t uErrCode;
12467	RTGCUINTPTR GCPtrFaultAddress;
12468	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12469	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12470	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12471	if (fErrorCodeValid)
12472	{
12473	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12474	AssertRCReturn(rc, rc);
12475	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12476	}
12477	else
12478	uErrCode = 0;
12479
12480	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12481	&& uVector == X86_XCPT_PF)
12482	GCPtrFaultAddress = pMixedCtx->cr2;
12483	else
12484	GCPtrFaultAddress = 0;
12485
12486	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12487	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12488
12489	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12490	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12491	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12492	}
12493	}
12494
12495	/* Fall back to the interpreter to emulate the task-switch. */
12496	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12497	return VERR_EM_INTERPRETER;
12498	}
12499
12500
12501	/**
12502	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12503	*/
12504	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12505	{
12506	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12507	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12508	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12509	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12510	AssertRCReturn(rc, rc);
12511	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12512	return VINF_EM_DBG_STEPPED;
12513	}
12514
12515
12516	/**
12517	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12518	*/
12519	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12520	{
12521	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12522
12523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12524
12525	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12526	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12527	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12528	{
12529	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12530	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12531	{
12532	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12533	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12534	}
12535	}
12536	else
12537	{
12538	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12539	rcStrict1 = VINF_SUCCESS;
12540	return rcStrict1;
12541	}
12542
12543	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12544	int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12545	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12546	AssertRCReturn(rc, rc);
12547
12548	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12549	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12550	VBOXSTRICTRC rcStrict2;
12551	switch (uAccessType)
12552	{
12553	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12554	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12555	{
12556	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12557	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12558	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12559
12560	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12561	GCPhys &= PAGE_BASE_GC_MASK;
12562	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12563	PVM pVM = pVCpu->CTX_SUFF(pVM);
12564	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12565	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12566
12567	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12568	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12569	CPUMCTX2CORE(pMixedCtx), GCPhys);
12570	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12571	if ( rcStrict2 == VINF_SUCCESS
12572	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12573	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12574	{
12575	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12576	\| HM_CHANGED_GUEST_RSP
12577	\| HM_CHANGED_GUEST_RFLAGS
12578	\| HM_CHANGED_GUEST_APIC_TPR);
12579	rcStrict2 = VINF_SUCCESS;
12580	}
12581	break;
12582	}
12583
12584	default:
12585	Log4Func(("uAccessType=%#x\n", uAccessType));
12586	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12587	break;
12588	}
12589
12590	if (rcStrict2 != VINF_SUCCESS)
12591	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12592	return rcStrict2;
12593	}
12594
12595
12596	/**
12597	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12598	* VM-exit.
12599	*/
12600	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12601	{
12602	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12603
12604	/* We should -not- get this VM-exit if the guest's debug registers were active. */
12605	if (pVmxTransient->fWasGuestDebugStateActive)
12606	{
12607	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12608	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12609	}
12610
12611	if ( !pVCpu->hm.s.fSingleInstruction
12612	&& !pVmxTransient->fWasHyperDebugStateActive)
12613	{
12614	Assert(!DBGFIsStepping(pVCpu));
12615	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12616
12617	/* Don't intercept MOV DRx any more. */
12618	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12619	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12620	AssertRCReturn(rc, rc);
12621
12622	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12623	VMMRZCallRing3Disable(pVCpu);
12624	HM_DISABLE_PREEMPT();
12625
12626	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12627	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12628	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
12629
12630	HM_RESTORE_PREEMPT();
12631	VMMRZCallRing3Enable(pVCpu);
12632
12633	#ifdef VBOX_WITH_STATISTICS
12634	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12635	AssertRCReturn(rc, rc);
12636	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12637	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12638	else
12639	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12640	#endif
12641	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12642	return VINF_SUCCESS;
12643	}
12644
12645	/*
12646	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12647	* Update the segment registers and DR7 from the CPU.
12648	*/
12649	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12650	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12651	\| CPUMCTX_EXTRN_DR7);
12652	AssertRCReturn(rc, rc);
12653	Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12654
12655	PVM pVM = pVCpu->CTX_SUFF(pVM);
12656	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12657	{
12658	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12659	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12660	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12661	if (RT_SUCCESS(rc))
12662	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12663	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12664	}
12665	else
12666	{
12667	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12668	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12669	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12670	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12671	}
12672
12673	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
12674	if (RT_SUCCESS(rc))
12675	{
12676	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12677	AssertRCReturn(rc2, rc2);
12678	return VINF_SUCCESS;
12679	}
12680	return rc;
12681	}
12682
12683
12684	/**
12685	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12686	* Conditional VM-exit.
12687	*/
12688	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12689	{
12690	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12691	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12692
12693	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12694	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12695	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12696	{
12697	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12698	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12699	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12700	{
12701	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12702	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12703	}
12704	}
12705	else
12706	{
12707	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12708	rcStrict1 = VINF_SUCCESS;
12709	return rcStrict1;
12710	}
12711
12712	/*
12713	* Get sufficent state and update the exit history entry.
12714	*/
12715	RTGCPHYS GCPhys;
12716	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12717	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12718	AssertRCReturn(rc, rc);
12719
12720	VBOXSTRICTRC rcStrict;
12721	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12722	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
12723	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12724	if (!pExitRec)
12725	{
12726	/*
12727	* If we succeed, resume guest execution.
12728	* If we fail in interpreting the instruction because we couldn't get the guest physical address
12729	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12730	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12731	* weird case. See @bugref{6043}.
12732	*/
12733	PVM pVM = pVCpu->CTX_SUFF(pVM);
12734	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12735	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12736	if ( rcStrict == VINF_SUCCESS
12737	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12738	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
12739	{
12740	/* Successfully handled MMIO operation. */
12741	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12742	\| HM_CHANGED_GUEST_RSP
12743	\| HM_CHANGED_GUEST_RFLAGS
12744	\| HM_CHANGED_GUEST_APIC_TPR);
12745	rcStrict = VINF_SUCCESS;
12746	}
12747	}
12748	else
12749	{
12750	/*
12751	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
12752	*/
12753	Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12754	int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12755	AssertRCReturn(rc2, rc2);
12756
12757	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12758	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12759
12760	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12761	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12762
12763	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12764	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12765	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12766	}
12767	return VBOXSTRICTRC_TODO(rcStrict);
12768	}
12769
12770
12771	/**
12772	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12773	* VM-exit.
12774	*/
12775	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12776	{
12777	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12778	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12779
12780	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12781	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12782	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12783	{
12784	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12785	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12786	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12787	}
12788	else
12789	{
12790	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12791	rcStrict1 = VINF_SUCCESS;
12792	return rcStrict1;
12793	}
12794
12795	RTGCPHYS GCPhys;
12796	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12797	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12798	rc \|= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12799	AssertRCReturn(rc, rc);
12800
12801	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12802	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12803
12804	RTGCUINT uErrorCode = 0;
12805	if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12806	uErrorCode \|= X86_TRAP_PF_ID;
12807	if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12808	uErrorCode \|= X86_TRAP_PF_RW;
12809	if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12810	uErrorCode \|= X86_TRAP_PF_P;
12811
12812	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12813
12814	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12815	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12816
12817	/* Handle the pagefault trap for the nested shadow table. */
12818	PVM pVM = pVCpu->CTX_SUFF(pVM);
12819	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12820	TRPMResetTrap(pVCpu);
12821
12822	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12823	if ( rcStrict2 == VINF_SUCCESS
12824	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12825	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12826	{
12827	/* Successfully synced our nested page tables. */
12828	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12829	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12830	\| HM_CHANGED_GUEST_RSP
12831	\| HM_CHANGED_GUEST_RFLAGS);
12832	return VINF_SUCCESS;
12833	}
12834
12835	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12836	return rcStrict2;
12837	}
12838
12839	/** @} */
12840
12841	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12842	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12843	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12844
12845	/** @name VM-exit exception handlers.
12846	* @{
12847	*/
12848
12849	/**
12850	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
12851	*/
12852	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12853	{
12854	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12855	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12856
12857	int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12858	AssertRCReturn(rc, rc);
12859
12860	if (!(pMixedCtx->cr0 & X86_CR0_NE))
12861	{
12862	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12863	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12864
12865	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12866	* provides VM-exit instruction length. If this causes problem later,
12867	* disassemble the instruction like it's done on AMD-V. */
12868	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12869	AssertRCReturn(rc2, rc2);
12870	return rc;
12871	}
12872
12873	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12874	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12875	return rc;
12876	}
12877
12878
12879	/**
12880	* VM-exit exception handler for \#BP (Breakpoint exception).
12881	*/
12882	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12883	{
12884	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12885	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12886
12887	int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12888	AssertRCReturn(rc, rc);
12889
12890	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12891	if (rc == VINF_EM_RAW_GUEST_TRAP)
12892	{
12893	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12894	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12895	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12896	AssertRCReturn(rc, rc);
12897
12898	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12899	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12900	}
12901
12902	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
12903	return rc;
12904	}
12905
12906
12907	/**
12908	* VM-exit exception handler for \#AC (alignment check exception).
12909	*/
12910	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12911	{
12912	RT_NOREF_PV(pMixedCtx);
12913	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12914
12915	/*
12916	* Re-inject it. We'll detect any nesting before getting here.
12917	*/
12918	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12919	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12920	AssertRCReturn(rc, rc);
12921	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12922
12923	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12924	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12925	return VINF_SUCCESS;
12926	}
12927
12928
12929	/**
12930	* VM-exit exception handler for \#DB (Debug exception).
12931	*/
12932	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12933	{
12934	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12935	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12936
12937	/*
12938	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12939	* for processing.
12940	*/
12941	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12942
12943	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12944	uint64_t uDR6 = X86_DR6_INIT_VAL;
12945	uDR6 \|= ( pVmxTransient->uExitQualification
12946	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
12947
12948	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12949	Log6Func(("rc=%Rrc\n", rc));
12950	if (rc == VINF_EM_RAW_GUEST_TRAP)
12951	{
12952	/*
12953	* The exception was for the guest. Update DR6, DR7.GD and
12954	* IA32_DEBUGCTL.LBR before forwarding it.
12955	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12956	*/
12957	VMMRZCallRing3Disable(pVCpu);
12958	HM_DISABLE_PREEMPT();
12959
12960	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12961	pMixedCtx->dr[6] \|= uDR6;
12962	if (CPUMIsGuestDebugStateActive(pVCpu))
12963	ASMSetDR6(pMixedCtx->dr[6]);
12964
12965	HM_RESTORE_PREEMPT();
12966	VMMRZCallRing3Enable(pVCpu);
12967
12968	rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12969	AssertRCReturn(rc, rc);
12970
12971	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12972	pMixedCtx->dr[7] &= ~X86_DR7_GD;
12973
12974	/* Paranoia. */
12975	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12976	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
12977
12978	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12979	AssertRCReturn(rc, rc);
12980
12981	/*
12982	* Raise #DB in the guest.
12983	*
12984	* It is important to reflect exactly what the VM-exit gave us (preserving the
12985	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12986	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12987	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12988	*
12989	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12990	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12991	*/
12992	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12993	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12994	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12995	AssertRCReturn(rc, rc);
12996	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12997	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12998	return VINF_SUCCESS;
12999	}
13000
13001	/*
13002	* Not a guest trap, must be a hypervisor related debug event then.
13003	* Update DR6 in case someone is interested in it.
13004	*/
13005	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13006	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13007	CPUMSetHyperDR6(pVCpu, uDR6);
13008
13009	return rc;
13010	}
13011
13012	/**
13013	* VM-exit exception handler for \#GP (General-protection exception).
13014	*
13015	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13016	*/
13017	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13018	{
13019	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13020	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13021
13022	int rc;
13023	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13024	{ /* likely */ }
13025	else
13026	{
13027	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13028	Assert(pVCpu->hm.s.fUsingDebugLoop);
13029	#endif
13030	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13031	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13032	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13033	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13034	rc \|= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13035	AssertRCReturn(rc, rc);
13036	Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13037	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13038	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13039	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13040	return rc;
13041	}
13042
13043	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13044	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13045
13046	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13047	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13048	AssertRCReturn(rc, rc);
13049
13050	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13051	uint32_t cbOp = 0;
13052	PVM pVM = pVCpu->CTX_SUFF(pVM);
13053	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13054	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13055	if (RT_SUCCESS(rc))
13056	{
13057	rc = VINF_SUCCESS;
13058	Assert(cbOp == pDis->cbInstr);
13059	Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13060	switch (pDis->pCurInstr->uOpcode)
13061	{
13062	case OP_CLI:
13063	{
13064	pMixedCtx->eflags.Bits.u1IF = 0;
13065	pMixedCtx->eflags.Bits.u1RF = 0;
13066	pMixedCtx->rip += pDis->cbInstr;
13067	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13068	if ( !fDbgStepping
13069	&& pMixedCtx->eflags.Bits.u1TF)
13070	{
13071	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13072	AssertRCReturn(rc, rc);
13073	}
13074	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13075	break;
13076	}
13077
13078	case OP_STI:
13079	{
13080	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13081	pMixedCtx->eflags.Bits.u1IF = 1;
13082	pMixedCtx->eflags.Bits.u1RF = 0;
13083	pMixedCtx->rip += pDis->cbInstr;
13084	if (!fOldIF)
13085	{
13086	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13087	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13088	}
13089	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13090	if ( !fDbgStepping
13091	&& pMixedCtx->eflags.Bits.u1TF)
13092	{
13093	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13094	AssertRCReturn(rc, rc);
13095	}
13096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13097	break;
13098	}
13099
13100	case OP_HLT:
13101	{
13102	rc = VINF_EM_HALT;
13103	pMixedCtx->rip += pDis->cbInstr;
13104	pMixedCtx->eflags.Bits.u1RF = 0;
13105	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13107	break;
13108	}
13109
13110	case OP_POPF:
13111	{
13112	Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13113	uint32_t cbParm;
13114	uint32_t uMask;
13115	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13116	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13117	{
13118	cbParm = 4;
13119	uMask = 0xffffffff;
13120	}
13121	else
13122	{
13123	cbParm = 2;
13124	uMask = 0xffff;
13125	}
13126
13127	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13128	RTGCPTR GCPtrStack = 0;
13129	X86EFLAGS Eflags;
13130	Eflags.u32 = 0;
13131	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13132	&GCPtrStack);
13133	if (RT_SUCCESS(rc))
13134	{
13135	Assert(sizeof(Eflags.u32) >= cbParm);
13136	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13137	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13138	}
13139	if (RT_FAILURE(rc))
13140	{
13141	rc = VERR_EM_INTERPRETER;
13142	break;
13143	}
13144	Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13145	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13146	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13147	pMixedCtx->esp += cbParm;
13148	pMixedCtx->esp &= uMask;
13149	pMixedCtx->rip += pDis->cbInstr;
13150	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13151	\| HM_CHANGED_GUEST_RSP
13152	\| HM_CHANGED_GUEST_RFLAGS);
13153	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13154	POPF restores EFLAGS.TF. */
13155	if ( !fDbgStepping
13156	&& fGstStepping)
13157	{
13158	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13159	AssertRCReturn(rc, rc);
13160	}
13161	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13162	break;
13163	}
13164
13165	case OP_PUSHF:
13166	{
13167	uint32_t cbParm;
13168	uint32_t uMask;
13169	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13170	{
13171	cbParm = 4;
13172	uMask = 0xffffffff;
13173	}
13174	else
13175	{
13176	cbParm = 2;
13177	uMask = 0xffff;
13178	}
13179
13180	/* Get the stack pointer & push the contents of eflags onto the stack. */
13181	RTGCPTR GCPtrStack = 0;
13182	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13183	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13184	if (RT_FAILURE(rc))
13185	{
13186	rc = VERR_EM_INTERPRETER;
13187	break;
13188	}
13189	X86EFLAGS Eflags = pMixedCtx->eflags;
13190	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13191	Eflags.Bits.u1RF = 0;
13192	Eflags.Bits.u1VM = 0;
13193
13194	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13195	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13196	{
13197	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13198	rc = VERR_EM_INTERPRETER;
13199	break;
13200	}
13201	Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13202	pMixedCtx->esp -= cbParm;
13203	pMixedCtx->esp &= uMask;
13204	pMixedCtx->rip += pDis->cbInstr;
13205	pMixedCtx->eflags.Bits.u1RF = 0;
13206	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13207	\| HM_CHANGED_GUEST_RSP
13208	\| HM_CHANGED_GUEST_RFLAGS);
13209	if ( !fDbgStepping
13210	&& pMixedCtx->eflags.Bits.u1TF)
13211	{
13212	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13213	AssertRCReturn(rc, rc);
13214	}
13215	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13216	break;
13217	}
13218
13219	case OP_IRET:
13220	{
13221	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13222	* instruction reference. */
13223	RTGCPTR GCPtrStack = 0;
13224	uint32_t uMask = 0xffff;
13225	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13226	uint16_t aIretFrame[3];
13227	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13228	{
13229	rc = VERR_EM_INTERPRETER;
13230	break;
13231	}
13232	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13233	&GCPtrStack);
13234	if (RT_SUCCESS(rc))
13235	{
13236	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13237	PGMACCESSORIGIN_HM));
13238	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13239	}
13240	if (RT_FAILURE(rc))
13241	{
13242	rc = VERR_EM_INTERPRETER;
13243	break;
13244	}
13245	pMixedCtx->eip = 0;
13246	pMixedCtx->ip = aIretFrame[0];
13247	pMixedCtx->cs.Sel = aIretFrame[1];
13248	pMixedCtx->cs.ValidSel = aIretFrame[1];
13249	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13250	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13251	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13252	pMixedCtx->sp += sizeof(aIretFrame);
13253	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13254	\| HM_CHANGED_GUEST_CS
13255	\| HM_CHANGED_GUEST_RSP
13256	\| HM_CHANGED_GUEST_RFLAGS);
13257	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13258	if ( !fDbgStepping
13259	&& fGstStepping)
13260	{
13261	rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13262	AssertRCReturn(rc, rc);
13263	}
13264	Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13265	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13266	break;
13267	}
13268
13269	case OP_INT:
13270	{
13271	uint16_t uVector = pDis->Param1.uValue & 0xff;
13272	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13273	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13274	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13275	break;
13276	}
13277
13278	case OP_INTO:
13279	{
13280	if (pMixedCtx->eflags.Bits.u1OF)
13281	{
13282	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13283	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13284	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13285	}
13286	else
13287	{
13288	pMixedCtx->eflags.Bits.u1RF = 0;
13289	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13290	}
13291	break;
13292	}
13293
13294	default:
13295	{
13296	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13297	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13298	EMCODETYPE_SUPERVISOR);
13299	rc = VBOXSTRICTRC_VAL(rc2);
13300	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13301	/** @todo We have to set pending-debug exceptions here when the guest is
13302	* single-stepping depending on the instruction that was interpreted. */
13303	Log4Func(("#GP rc=%Rrc\n", rc));
13304	break;
13305	}
13306	}
13307	}
13308	else
13309	rc = VERR_EM_INTERPRETER;
13310
13311	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13312	("#GP Unexpected rc=%Rrc\n", rc));
13313	return rc;
13314	}
13315
13316
13317	/**
13318	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13319	* the exception reported in the VMX transient structure back into the VM.
13320	*
13321	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13322	* up-to-date.
13323	*/
13324	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13325	{
13326	RT_NOREF_PV(pMixedCtx);
13327	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13328	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13329	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13330	("uVector=%#x u32XcptBitmap=%#X32\n",
13331	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13332	#endif
13333
13334	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13335	hmR0VmxCheckExitDueToEventDelivery(). */
13336	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13337	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13338	AssertRCReturn(rc, rc);
13339	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13340
13341	#ifdef DEBUG_ramshankar
13342	rc \|= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13343	\| CPUMCTX_EXTRN_RIP);
13344	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13345	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13346	#endif
13347
13348	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13349	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13350	return VINF_SUCCESS;
13351	}
13352
13353
13354	/**
13355	* VM-exit exception handler for \#PF (Page-fault exception).
13356	*/
13357	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13358	{
13359	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13360	PVM pVM = pVCpu->CTX_SUFF(pVM);
13361	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13362	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13363	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13364	AssertRCReturn(rc, rc);
13365
13366	if (!pVM->hm.s.fNestedPaging)
13367	{ /* likely */ }
13368	else
13369	{
13370	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13371	Assert(pVCpu->hm.s.fUsingDebugLoop);
13372	#endif
13373	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13374	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13375	{
13376	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13377	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13378	}
13379	else
13380	{
13381	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13382	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13383	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13384	}
13385	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13386	return rc;
13387	}
13388
13389	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13390	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13391	if (pVmxTransient->fVectoringPF)
13392	{
13393	Assert(pVCpu->hm.s.Event.fPending);
13394	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13395	}
13396
13397	rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13398	AssertRCReturn(rc, rc);
13399
13400	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13401	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13402
13403	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13404	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13405	(RTGCPTR)pVmxTransient->uExitQualification);
13406
13407	Log4Func(("#PF: rc=%Rrc\n", rc));
13408	if (rc == VINF_SUCCESS)
13409	{
13410	/*
13411	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13412	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13413	*/
13414	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13415	TRPMResetTrap(pVCpu);
13416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13417	return rc;
13418	}
13419
13420	if (rc == VINF_EM_RAW_GUEST_TRAP)
13421	{
13422	if (!pVmxTransient->fVectoringDoublePF)
13423	{
13424	/* It's a guest page fault and needs to be reflected to the guest. */
13425	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13426	TRPMResetTrap(pVCpu);
13427	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13428	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13429	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13430	}
13431	else
13432	{
13433	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13434	TRPMResetTrap(pVCpu);
13435	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13436	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13437	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13438	}
13439
13440	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13441	return VINF_SUCCESS;
13442	}
13443
13444	TRPMResetTrap(pVCpu);
13445	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13446	return rc;
13447	}
13448
13449	/** @} */
13450

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72872

Download in other formats: