HMVMXR0.cpp@ 47802

Last change on this file since 47802 was 47802, checked in by vboxsync, 11 years ago
VMM/HM: Redundant updates of flags is not necessary.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 434.3 KB

Line
1	/* $Id: HMVMXR0.cpp 47802 2013-08-16 11:27:27Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#endif
45
46
47	/*******************************************************************************
48	* Defined Constants And Macros *
49	*******************************************************************************/
50	#if defined(RT_ARCH_AMD64)
51	# define HMVMX_IS_64BIT_HOST_MODE() (true)
52	typedef RTHCUINTREG HMVMXHCUINTREG;
53	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54	extern "C" uint32_t g_fVMXIs64bitHost;
55	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56	typedef uint64_t HMVMXHCUINTREG;
57	#else
58	# define HMVMX_IS_64BIT_HOST_MODE() (false)
59	typedef RTHCUINTREG HMVMXHCUINTREG;
60	#endif
61
62	/** Use the function table. */
63	#define HMVMX_USE_FUNCTION_TABLE
64
65	/** Determine which tagged-TLB flush handler to use. */
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71	/** @name Updated-guest-state flags.
72	* @{ */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113	/** @} */
114
115	/** @name
116	* Flags to skip redundant reads of some common VMCS fields that are not part of
117	* the guest-CPU state but are in the transient structure.
118	*/
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125	/** @} */
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86).
129	*
130	* We need to intercept all exceptions manually (except #PF). #NM is also
131	* handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
132	* intercepted even in real-mode if we have Nested Paging support.
133	*/
134	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
135	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
136	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
137	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
138	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
139	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
140	\| RT_BIT(X86_XCPT_XF))
141
142	/**
143	* Exception bitmap mask for all contributory exceptions.
144	*
145	* Page fault is deliberately excluded here as it's conditional as to whether
146	* it's contributory or benign. Page faults are handled separately.
147	*/
148	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
149	\| RT_BIT(X86_XCPT_DE))
150
151	/** Maximum VM-instruction error number. */
152	#define HMVMX_INSTR_ERROR_MAX 28
153
154	/** Profiling macro. */
155	#ifdef HM_PROFILE_EXIT_DISPATCH
156	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
157	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
158	#else
159	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
160	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
161	#endif
162
163
164	/*******************************************************************************
165	* Structures and Typedefs *
166	*******************************************************************************/
167	/**
168	* VMX transient state.
169	*
170	* A state structure for holding miscellaneous information across
171	* VMX non-root operation and restored after the transition.
172	*/
173	typedef struct VMXTRANSIENT
174	{
175	/** The host's rflags/eflags. */
176	RTCCUINTREG uEflags;
177	#if HC_ARCH_BITS == 32
178	uint32_t u32Alignment0;
179	#endif
180	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
181	uint64_t u64LStarMsr;
182	/** The guest's TPR value used for TPR shadowing. */
183	uint8_t u8GuestTpr;
184	/** Alignment. */
185	uint8_t abAlignment0[7];
186
187	/** The basic VM-exit reason. */
188	uint16_t uExitReason;
189	/** Alignment. */
190	uint16_t u16Alignment0;
191	/** The VM-exit interruption error code. */
192	uint32_t uExitIntrErrorCode;
193	/** The VM-exit exit qualification. */
194	uint64_t uExitQualification;
195
196	/** The VM-exit interruption-information field. */
197	uint32_t uExitIntrInfo;
198	/** The VM-exit instruction-length field. */
199	uint32_t cbInstr;
200	/** The VM-exit instruction-information field. */
201	union
202	{
203	/** Plain unsigned int representation. */
204	uint32_t u;
205	/** INS and OUTS information. */
206	struct
207	{
208	uint32_t u6Reserved0 : 6;
209	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
210	uint32_t u3AddrSize : 3;
211	uint32_t u5Reserved1 : 5;
212	/** The segment register (X86_SREG_XXX). */
213	uint32_t iSegReg : 3;
214	uint32_t uReserved2 : 14;
215	} StrIo;
216	} ExitInstrInfo;
217	/** Whether the VM-entry failed or not. */
218	bool fVMEntryFailed;
219	/** Alignment. */
220	uint8_t abAlignment1[3];
221
222	/** The VM-entry interruption-information field. */
223	uint32_t uEntryIntrInfo;
224	/** The VM-entry exception error code field. */
225	uint32_t uEntryXcptErrorCode;
226	/** The VM-entry instruction length field. */
227	uint32_t cbEntryInstr;
228
229	/** IDT-vectoring information field. */
230	uint32_t uIdtVectoringInfo;
231	/** IDT-vectoring error code. */
232	uint32_t uIdtVectoringErrorCode;
233
234	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
235	uint32_t fVmcsFieldsRead;
236	/** Whether TSC-offsetting should be setup before VM-entry. */
237	bool fUpdateTscOffsettingAndPreemptTimer;
238	/** Whether the VM-exit was caused by a page-fault during delivery of a
239	* contributory exception or a page-fault. */
240	bool fVectoringPF;
241	} VMXTRANSIENT;
242	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
243	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
244	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
245	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
246	/** Pointer to VMX transient state. */
247	typedef VMXTRANSIENT *PVMXTRANSIENT;
248
249
250	/**
251	* MSR-bitmap read permissions.
252	*/
253	typedef enum VMXMSREXITREAD
254	{
255	/** Reading this MSR causes a VM-exit. */
256	VMXMSREXIT_INTERCEPT_READ = 0xb,
257	/** Reading this MSR does not cause a VM-exit. */
258	VMXMSREXIT_PASSTHRU_READ
259	} VMXMSREXITREAD;
260
261	/**
262	* MSR-bitmap write permissions.
263	*/
264	typedef enum VMXMSREXITWRITE
265	{
266	/** Writing to this MSR causes a VM-exit. */
267	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
268	/** Writing to this MSR does not cause a VM-exit. */
269	VMXMSREXIT_PASSTHRU_WRITE
270	} VMXMSREXITWRITE;
271
272	/**
273	* VM-exit handler.
274	*
275	* @returns VBox status code.
276	* @param pVCpu Pointer to the VMCPU.
277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
278	* out-of-sync. Make sure to update the required
279	* fields before using them.
280	* @param pVmxTransient Pointer to the VMX-transient structure.
281	*/
282	#ifndef HMVMX_USE_FUNCTION_TABLE
283	typedef int FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	#else
285	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	/** Pointer to VM-exit handler. */
287	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
288	#endif
289
290
291	/*******************************************************************************
292	* Internal Functions *
293	*******************************************************************************/
294	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
295	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
296	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
297	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
298	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
299	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
300	#endif
301	#ifndef HMVMX_USE_FUNCTION_TABLE
302	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
303	# define HMVMX_EXIT_DECL static int
304	#else
305	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
306	#endif
307
308	/** @name VM-exit handlers.
309	* @{
310	*/
311	static FNVMEXITHANDLER hmR0VmxExitXcptOrNmi;
312	static FNVMEXITHANDLER hmR0VmxExitExtInt;
313	static FNVMEXITHANDLER hmR0VmxExitTripleFault;
314	static FNVMEXITHANDLER hmR0VmxExitInitSignal;
315	static FNVMEXITHANDLER hmR0VmxExitSipi;
316	static FNVMEXITHANDLER hmR0VmxExitIoSmi;
317	static FNVMEXITHANDLER hmR0VmxExitSmi;
318	static FNVMEXITHANDLER hmR0VmxExitIntWindow;
319	static FNVMEXITHANDLER hmR0VmxExitNmiWindow;
320	static FNVMEXITHANDLER hmR0VmxExitTaskSwitch;
321	static FNVMEXITHANDLER hmR0VmxExitCpuid;
322	static FNVMEXITHANDLER hmR0VmxExitGetsec;
323	static FNVMEXITHANDLER hmR0VmxExitHlt;
324	static FNVMEXITHANDLER hmR0VmxExitInvd;
325	static FNVMEXITHANDLER hmR0VmxExitInvlpg;
326	static FNVMEXITHANDLER hmR0VmxExitRdpmc;
327	static FNVMEXITHANDLER hmR0VmxExitRdtsc;
328	static FNVMEXITHANDLER hmR0VmxExitRsm;
329	static FNVMEXITHANDLER hmR0VmxExitSetPendingXcptUD;
330	static FNVMEXITHANDLER hmR0VmxExitMovCRx;
331	static FNVMEXITHANDLER hmR0VmxExitMovDRx;
332	static FNVMEXITHANDLER hmR0VmxExitIoInstr;
333	static FNVMEXITHANDLER hmR0VmxExitRdmsr;
334	static FNVMEXITHANDLER hmR0VmxExitWrmsr;
335	static FNVMEXITHANDLER hmR0VmxExitErrInvalidGuestState;
336	static FNVMEXITHANDLER hmR0VmxExitErrMsrLoad;
337	static FNVMEXITHANDLER hmR0VmxExitErrUndefined;
338	static FNVMEXITHANDLER hmR0VmxExitMwait;
339	static FNVMEXITHANDLER hmR0VmxExitMtf;
340	static FNVMEXITHANDLER hmR0VmxExitMonitor;
341	static FNVMEXITHANDLER hmR0VmxExitPause;
342	static FNVMEXITHANDLER hmR0VmxExitErrMachineCheck;
343	static FNVMEXITHANDLER hmR0VmxExitTprBelowThreshold;
344	static FNVMEXITHANDLER hmR0VmxExitApicAccess;
345	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
346	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
347	static FNVMEXITHANDLER hmR0VmxExitEptViolation;
348	static FNVMEXITHANDLER hmR0VmxExitEptMisconfig;
349	static FNVMEXITHANDLER hmR0VmxExitRdtscp;
350	static FNVMEXITHANDLER hmR0VmxExitPreemptTimer;
351	static FNVMEXITHANDLER hmR0VmxExitWbinvd;
352	static FNVMEXITHANDLER hmR0VmxExitXsetbv;
353	static FNVMEXITHANDLER hmR0VmxExitRdrand;
354	static FNVMEXITHANDLER hmR0VmxExitInvpcid;
355	/** @} */
356
357	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
358	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
359	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
360	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
361	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
362	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
363	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
365
366	/*******************************************************************************
367	* Global Variables *
368	*******************************************************************************/
369	#ifdef HMVMX_USE_FUNCTION_TABLE
370
371	/**
372	* VMX_EXIT dispatch table.
373	*/
374	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
375	{
376	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
377	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
378	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
379	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
380	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
381	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
382	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
383	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
384	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
385	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
386	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
387	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
388	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
389	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
390	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
391	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
392	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
393	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
394	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
395	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
396	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
397	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
398	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
399	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
400	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
401	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
402	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
403	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
404	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
405	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
406	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
407	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
408	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
409	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
410	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
411	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
412	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
413	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
414	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
415	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
416	/* 40 UNDEFINED */ hmR0VmxExitPause,
417	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
418	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
419	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
420	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
421	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
422	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
423	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
424	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
425	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
426	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
427	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
428	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
429	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
430	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
431	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
432	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
433	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
434	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
435	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
436	};
437	#endif /* HMVMX_USE_FUNCTION_TABLE */
438
439	#ifdef VBOX_STRICT
440	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
441	{
442	/* 0 */ "(Not Used)",
443	/* 1 */ "VMCALL executed in VMX root operation.",
444	/* 2 */ "VMCLEAR with invalid physical address.",
445	/* 3 */ "VMCLEAR with VMXON pointer.",
446	/* 4 */ "VMLAUNCH with non-clear VMCS.",
447	/* 5 */ "VMRESUME with non-launched VMCS.",
448	/* 6 */ "VMRESUME after VMXOFF",
449	/* 7 */ "VM entry with invalid control fields.",
450	/* 8 */ "VM entry with invalid host state fields.",
451	/* 9 */ "VMPTRLD with invalid physical address.",
452	/* 10 */ "VMPTRLD with VMXON pointer.",
453	/* 11 */ "VMPTRLD with incorrect revision identifier.",
454	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
455	/* 13 */ "VMWRITE to read-only VMCS component.",
456	/* 14 */ "(Not Used)",
457	/* 15 */ "VMXON executed in VMX root operation.",
458	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
459	/* 17 */ "VM entry with non-launched executing VMCS.",
460	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
461	/* 19 */ "VMCALL with non-clear VMCS.",
462	/* 20 */ "VMCALL with invalid VM-exit control fields.",
463	/* 21 */ "(Not Used)",
464	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
465	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
466	/* 24 */ "VMCALL with invalid SMM-monitor features.",
467	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
468	/* 26 */ "VM entry with events blocked by MOV SS.",
469	/* 27 */ "(Not Used)",
470	/* 28 */ "Invalid operand to INVEPT/INVVPID."
471	};
472	#endif /* VBOX_STRICT */
473
474
475
476	/**
477	* Updates the VM's last error record. If there was a VMX instruction error,
478	* reads the error data from the VMCS and updates VCPU's last error record as
479	* well.
480	*
481	* @param pVM Pointer to the VM.
482	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
483	* VERR_VMX_UNABLE_TO_START_VM or
484	* VERR_VMX_INVALID_VMCS_FIELD).
485	* @param rc The error code.
486	*/
487	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
488	{
489	AssertPtr(pVM);
490	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
491	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
492	{
493	AssertPtrReturnVoid(pVCpu);
494	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
495	}
496	pVM->hm.s.lLastError = rc;
497	}
498
499
500	/**
501	* Reads the VM-entry interruption-information field from the VMCS into the VMX
502	* transient structure.
503	*
504	* @returns VBox status code.
505	* @param pVmxTransient Pointer to the VMX transient structure.
506	*
507	* @remarks No-long-jump zone!!!
508	*/
509	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
510	{
511	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
512	AssertRCReturn(rc, rc);
513	return VINF_SUCCESS;
514	}
515
516
517	/**
518	* Reads the VM-entry exception error code field from the VMCS into
519	* the VMX transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVmxTransient Pointer to the VMX transient structure.
523	*
524	* @remarks No-long-jump zone!!!
525	*/
526	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
527	{
528	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
529	AssertRCReturn(rc, rc);
530	return VINF_SUCCESS;
531	}
532
533
534	/**
535	* Reads the VM-entry exception error code field from the VMCS into
536	* the VMX transient structure.
537	*
538	* @returns VBox status code.
539	* @param pVCpu Pointer to the VMCPU.
540	* @param pVmxTransient Pointer to the VMX transient structure.
541	*
542	* @remarks No-long-jump zone!!!
543	*/
544	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
545	{
546	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
547	AssertRCReturn(rc, rc);
548	return VINF_SUCCESS;
549	}
550
551
552	/**
553	* Reads the VM-exit interruption-information field from the VMCS into the VMX
554	* transient structure.
555	*
556	* @returns VBox status code.
557	* @param pVCpu Pointer to the VMCPU.
558	* @param pVmxTransient Pointer to the VMX transient structure.
559	*/
560	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
561	{
562	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
563	{
564	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
565	AssertRCReturn(rc, rc);
566	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
567	}
568	return VINF_SUCCESS;
569	}
570
571
572	/**
573	* Reads the VM-exit interruption error code from the VMCS into the VMX
574	* transient structure.
575	*
576	* @returns VBox status code.
577	* @param pVCpu Pointer to the VMCPU.
578	* @param pVmxTransient Pointer to the VMX transient structure.
579	*/
580	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
581	{
582	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
583	{
584	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
585	AssertRCReturn(rc, rc);
586	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
587	}
588	return VINF_SUCCESS;
589	}
590
591
592	/**
593	* Reads the VM-exit instruction length field from the VMCS into the VMX
594	* transient structure.
595	*
596	* @returns VBox status code.
597	* @param pVCpu Pointer to the VMCPU.
598	* @param pVmxTransient Pointer to the VMX transient structure.
599	*/
600	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
601	{
602	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
603	{
604	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
605	AssertRCReturn(rc, rc);
606	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
607	}
608	return VINF_SUCCESS;
609	}
610
611
612	/**
613	* Reads the VM-exit instruction-information field from the VMCS into
614	* the VMX transient structure.
615	*
616	* @returns VBox status code.
617	* @param pVCpu The cross context per CPU structure.
618	* @param pVmxTransient Pointer to the VMX transient structure.
619	*/
620	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
621	{
622	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
623	{
624	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
625	AssertRCReturn(rc, rc);
626	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
627	}
628	return VINF_SUCCESS;
629	}
630
631
632	/**
633	* Reads the exit qualification from the VMCS into the VMX transient structure.
634	*
635	* @returns VBox status code.
636	* @param pVCpu Pointer to the VMCPU.
637	* @param pVmxTransient Pointer to the VMX transient structure.
638	*/
639	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
640	{
641	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
642	{
643	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
644	AssertRCReturn(rc, rc);
645	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
646	}
647	return VINF_SUCCESS;
648	}
649
650
651	/**
652	* Reads the IDT-vectoring information field from the VMCS into the VMX
653	* transient structure.
654	*
655	* @returns VBox status code.
656	* @param pVmxTransient Pointer to the VMX transient structure.
657	*
658	* @remarks No-long-jump zone!!!
659	*/
660	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
661	{
662	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
663	{
664	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
665	AssertRCReturn(rc, rc);
666	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
667	}
668	return VINF_SUCCESS;
669	}
670
671
672	/**
673	* Reads the IDT-vectoring error code from the VMCS into the VMX
674	* transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVmxTransient Pointer to the VMX transient structure.
678	*/
679	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
680	{
681	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
682	{
683	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
684	AssertRCReturn(rc, rc);
685	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
686	}
687	return VINF_SUCCESS;
688	}
689
690
691	/**
692	* Enters VMX root mode operation on the current CPU.
693	*
694	* @returns VBox status code.
695	* @param pVM Pointer to the VM (optional, can be NULL, after
696	* a resume).
697	* @param HCPhysCpuPage Physical address of the VMXON region.
698	* @param pvCpuPage Pointer to the VMXON region.
699	*/
700	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
701	{
702	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
703	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
704	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
705
706	if (pVM)
707	{
708	/* Write the VMCS revision dword to the VMXON region. */
709	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
710	}
711
712	/* Enable the VMX bit in CR4 if necessary. */
713	RTCCUINTREG uCr4 = ASMGetCR4();
714	if (!(uCr4 & X86_CR4_VMXE))
715	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
716
717	/* Enter VMX root mode. */
718	int rc = VMXEnable(HCPhysCpuPage);
719	if (RT_FAILURE(rc))
720	ASMSetCR4(uCr4);
721
722	return rc;
723	}
724
725
726	/**
727	* Exits VMX root mode operation on the current CPU.
728	*
729	* @returns VBox status code.
730	*/
731	static int hmR0VmxLeaveRootMode(void)
732	{
733	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
734
735	/* If we're for some reason not in VMX root mode, then don't leave it. */
736	RTCCUINTREG uHostCR4 = ASMGetCR4();
737	if (uHostCR4 & X86_CR4_VMXE)
738	{
739	/* Exit VMX root mode and clear the VMX bit in CR4. */
740	VMXDisable();
741	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
742	return VINF_SUCCESS;
743	}
744
745	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
746	}
747
748
749	/**
750	* Allocates and maps one physically contiguous page. The allocated page is
751	* zero'd out. (Used by various VT-x structures).
752	*
753	* @returns IPRT status code.
754	* @param pMemObj Pointer to the ring-0 memory object.
755	* @param ppVirt Where to store the virtual address of the
756	* allocation.
757	* @param pPhys Where to store the physical address of the
758	* allocation.
759	*/
760	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
761	{
762	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
763	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
764	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
765
766	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
767	if (RT_FAILURE(rc))
768	return rc;
769	ppVirt = RTR0MemObjAddress(pMemObj);
770	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
771	ASMMemZero32(*ppVirt, PAGE_SIZE);
772	return VINF_SUCCESS;
773	}
774
775
776	/**
777	* Frees and unmaps an allocated physical page.
778	*
779	* @param pMemObj Pointer to the ring-0 memory object.
780	* @param ppVirt Where to re-initialize the virtual address of
781	* allocation as 0.
782	* @param pHCPhys Where to re-initialize the physical address of the
783	* allocation as 0.
784	*/
785	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
786	{
787	AssertPtr(pMemObj);
788	AssertPtr(ppVirt);
789	AssertPtr(pHCPhys);
790	if (*pMemObj != NIL_RTR0MEMOBJ)
791	{
792	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
793	AssertRC(rc);
794	*pMemObj = NIL_RTR0MEMOBJ;
795	*ppVirt = 0;
796	*pHCPhys = 0;
797	}
798	}
799
800
801	/**
802	* Worker function to free VT-x related structures.
803	*
804	* @returns IPRT status code.
805	* @param pVM Pointer to the VM.
806	*/
807	static void hmR0VmxStructsFree(PVM pVM)
808	{
809	for (VMCPUID i = 0; i < pVM->cCpus; i++)
810	{
811	PVMCPU pVCpu = &pVM->aCpus[i];
812	AssertPtr(pVCpu);
813
814	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
815	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
816	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
817	#endif
818
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
820	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
821
822	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
823	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
824	}
825
826	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
827	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
828	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
829	#endif
830	}
831
832
833	/**
834	* Worker function to allocate VT-x related VM structures.
835	*
836	* @returns IPRT status code.
837	* @param pVM Pointer to the VM.
838	*/
839	static int hmR0VmxStructsAlloc(PVM pVM)
840	{
841	/*
842	* Initialize members up-front so we can cleanup properly on allocation failure.
843	*/
844	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
845	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
846	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
847	pVM->hm.s.vmx.HCPhys##a_Name = 0;
848
849	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
850	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
851	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
852	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
853
854	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
855	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
856	#endif
857	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
858
859	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
860	for (VMCPUID i = 0; i < pVM->cCpus; i++)
861	{
862	PVMCPU pVCpu = &pVM->aCpus[i];
863	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
864	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
865	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
866	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
867	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
868	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
869	#endif
870	}
871	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
872	#undef VMXLOCAL_INIT_VM_MEMOBJ
873
874	/*
875	* Allocate all the VT-x structures.
876	*/
877	int rc = VINF_SUCCESS;
878	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
879	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
883	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
884	#endif
885
886	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
887	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
888	{
889	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
890	&pVM->hm.s.vmx.HCPhysApicAccess);
891	if (RT_FAILURE(rc))
892	goto cleanup;
893	}
894
895	/*
896	* Initialize per-VCPU VT-x structures.
897	*/
898	for (VMCPUID i = 0; i < pVM->cCpus; i++)
899	{
900	PVMCPU pVCpu = &pVM->aCpus[i];
901	AssertPtr(pVCpu);
902
903	/* Allocate the VM control structure (VMCS). */
904	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
905	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
906	if (RT_FAILURE(rc))
907	goto cleanup;
908
909	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
910	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
911	{
912	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
913	&pVCpu->hm.s.vmx.HCPhysVirtApic);
914	if (RT_FAILURE(rc))
915	goto cleanup;
916	}
917
918	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
919	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
920	{
921	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
922	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
923	if (RT_FAILURE(rc))
924	goto cleanup;
925	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
926	}
927
928	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
929	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
930	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
931	if (RT_FAILURE(rc))
932	goto cleanup;
933
934	/* Allocate the VM-exit MSR-load page for the host MSRs. */
935	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
936	if (RT_FAILURE(rc))
937	goto cleanup;
938	#endif
939	}
940
941	return VINF_SUCCESS;
942
943	cleanup:
944	hmR0VmxStructsFree(pVM);
945	return rc;
946	}
947
948
949	/**
950	* Does global VT-x initialization (called during module initialization).
951	*
952	* @returns VBox status code.
953	*/
954	VMMR0DECL(int) VMXR0GlobalInit(void)
955	{
956	#ifdef HMVMX_USE_FUNCTION_TABLE
957	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
958	# ifdef VBOX_STRICT
959	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
960	Assert(g_apfnVMExitHandlers[i]);
961	# endif
962	#endif
963	return VINF_SUCCESS;
964	}
965
966
967	/**
968	* Does global VT-x termination (called during module termination).
969	*/
970	VMMR0DECL(void) VMXR0GlobalTerm()
971	{
972	/* Nothing to do currently. */
973	}
974
975
976	/**
977	* Sets up and activates VT-x on the current CPU.
978	*
979	* @returns VBox status code.
980	* @param pCpu Pointer to the global CPU info struct.
981	* @param pVM Pointer to the VM (can be NULL after a host resume
982	* operation).
983	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
984	* fEnabledByHost is true).
985	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
986	* @a fEnabledByHost is true).
987	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
988	* enable VT-x on the host.
989	*/
990	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
991	{
992	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
993	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
994
995	if (!fEnabledByHost)
996	{
997	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
998	if (RT_FAILURE(rc))
999	return rc;
1000	}
1001
1002	/*
1003	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
1004	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
1005	* each time while reusing a VPID after hitting the MaxASID limit once.
1006	*/
1007	if ( pVM
1008	&& pVM->hm.s.fNestedPaging)
1009	{
1010	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
1011	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
1012	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1013	pCpu->fFlushAsidBeforeUse = false;
1014	}
1015	else
1016	{
1017	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
1018	* without Nested Paging triggered this function) we still have the risk
1019	* of potentially running with stale TLB-entries from other hypervisors
1020	* when later we use a VM with NestedPaging. To fix this properly we will
1021	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
1022	* 'vmx_ept_vpid_caps' from it. Sigh. */
1023	pCpu->fFlushAsidBeforeUse = true;
1024	}
1025
1026	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1027	++pCpu->cTlbFlushes;
1028
1029	return VINF_SUCCESS;
1030	}
1031
1032
1033	/**
1034	* Deactivates VT-x on the current CPU.
1035	*
1036	* @returns VBox status code.
1037	* @param pCpu Pointer to the global CPU info struct.
1038	* @param pvCpuPage Pointer to the VMXON region.
1039	* @param HCPhysCpuPage Physical address of the VMXON region.
1040	*
1041	* @remarks This function should never be called when SUPR0EnableVTx() or
1042	* similar was used to enable VT-x on the host.
1043	*/
1044	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1045	{
1046	NOREF(pCpu);
1047	NOREF(pvCpuPage);
1048	NOREF(HCPhysCpuPage);
1049
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051	return hmR0VmxLeaveRootMode();
1052	}
1053
1054
1055	/**
1056	* Sets the permission bits for the specified MSR in the MSR bitmap.
1057	*
1058	* @param pVCpu Pointer to the VMCPU.
1059	* @param uMSR The MSR value.
1060	* @param enmRead Whether reading this MSR causes a VM-exit.
1061	* @param enmWrite Whether writing this MSR causes a VM-exit.
1062	*/
1063	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1064	{
1065	int32_t iBit;
1066	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1067
1068	/*
1069	* Layout:
1070	* 0x000 - 0x3ff - Low MSR read bits
1071	* 0x400 - 0x7ff - High MSR read bits
1072	* 0x800 - 0xbff - Low MSR write bits
1073	* 0xc00 - 0xfff - High MSR write bits
1074	*/
1075	if (uMsr <= 0x00001FFF)
1076	iBit = uMsr;
1077	else if ( uMsr >= 0xC0000000
1078	&& uMsr <= 0xC0001FFF)
1079	{
1080	iBit = (uMsr - 0xC0000000);
1081	pbMsrBitmap += 0x400;
1082	}
1083	else
1084	{
1085	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1086	return;
1087	}
1088
1089	Assert(iBit <= 0x1fff);
1090	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1091	ASMBitSet(pbMsrBitmap, iBit);
1092	else
1093	ASMBitClear(pbMsrBitmap, iBit);
1094
1095	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1096	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1097	else
1098	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1099	}
1100
1101
1102	/**
1103	* Flushes the TLB using EPT.
1104	*
1105	* @returns VBox status code.
1106	* @param pVM Pointer to the VM.
1107	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1108	* enmFlush).
1109	* @param enmFlush Type of flush.
1110	*/
1111	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1112	{
1113	AssertPtr(pVM);
1114	Assert(pVM->hm.s.fNestedPaging);
1115
1116	uint64_t descriptor[2];
1117	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1118	descriptor[0] = 0;
1119	else
1120	{
1121	Assert(pVCpu);
1122	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1123	}
1124	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1125
1126	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1127	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1128	rc));
1129	if ( RT_SUCCESS(rc)
1130	&& pVCpu)
1131	{
1132	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1133	}
1134	}
1135
1136
1137	/**
1138	* Flushes the TLB using VPID.
1139	*
1140	* @returns VBox status code.
1141	* @param pVM Pointer to the VM.
1142	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1143	* enmFlush).
1144	* @param enmFlush Type of flush.
1145	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1146	* on @a enmFlush).
1147	*/
1148	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1149	{
1150	AssertPtr(pVM);
1151	Assert(pVM->hm.s.vmx.fVpid);
1152
1153	uint64_t descriptor[2];
1154	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1155	{
1156	descriptor[0] = 0;
1157	descriptor[1] = 0;
1158	}
1159	else
1160	{
1161	AssertPtr(pVCpu);
1162	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1163	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1164	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1165	descriptor[1] = GCPtr;
1166	}
1167
1168	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1169	AssertMsg(rc == VINF_SUCCESS,
1170	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1171	if ( RT_SUCCESS(rc)
1172	&& pVCpu)
1173	{
1174	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1175	}
1176	}
1177
1178
1179	/**
1180	* Invalidates a guest page by guest virtual address. Only relevant for
1181	* EPT/VPID, otherwise there is nothing really to invalidate.
1182	*
1183	* @returns VBox status code.
1184	* @param pVM Pointer to the VM.
1185	* @param pVCpu Pointer to the VMCPU.
1186	* @param GCVirt Guest virtual address of the page to invalidate.
1187	*/
1188	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1189	{
1190	AssertPtr(pVM);
1191	AssertPtr(pVCpu);
1192	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1193
1194	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1195	if (!fFlushPending)
1196	{
1197	/*
1198	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1199	* See @bugref{6043} and @bugref{6177}.
1200	*
1201	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1202	* function maybe called in a loop with individual addresses.
1203	*/
1204	if (pVM->hm.s.vmx.fVpid)
1205	{
1206	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1207	{
1208	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1209	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1210	}
1211	else
1212	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1213	}
1214	else if (pVM->hm.s.fNestedPaging)
1215	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1216	}
1217
1218	return VINF_SUCCESS;
1219	}
1220
1221
1222	/**
1223	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1224	* otherwise there is nothing really to invalidate.
1225	*
1226	* @returns VBox status code.
1227	* @param pVM Pointer to the VM.
1228	* @param pVCpu Pointer to the VMCPU.
1229	* @param GCPhys Guest physical address of the page to invalidate.
1230	*/
1231	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1232	{
1233	LogFlowFunc(("%RGp\n", GCPhys));
1234
1235	/*
1236	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1237	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1238	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1239	*/
1240	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1241	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1242	return VINF_SUCCESS;
1243	}
1244
1245
1246	/**
1247	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1248	* case where neither EPT nor VPID is supported by the CPU.
1249	*
1250	* @param pVM Pointer to the VM.
1251	* @param pVCpu Pointer to the VMCPU.
1252	*
1253	* @remarks Called with interrupts disabled.
1254	*/
1255	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1256	{
1257	NOREF(pVM);
1258	AssertPtr(pVCpu);
1259	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1260	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1261
1262	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1263	AssertPtr(pCpu);
1264
1265	pVCpu->hm.s.TlbShootdown.cPages = 0;
1266	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1267	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1268	pVCpu->hm.s.fForceTLBFlush = false;
1269	return;
1270	}
1271
1272
1273	/**
1274	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1275	*
1276	* @param pVM Pointer to the VM.
1277	* @param pVCpu Pointer to the VMCPU.
1278	* @remarks All references to "ASID" in this function pertains to "VPID" in
1279	* Intel's nomenclature. The reason is, to avoid confusion in compare
1280	* statements since the host-CPU copies are named "ASID".
1281	*
1282	* @remarks Called with interrupts disabled.
1283	*/
1284	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1285	{
1286	#ifdef VBOX_WITH_STATISTICS
1287	bool fTlbFlushed = false;
1288	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1289	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1290	if (!fTlbFlushed) \
1291	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1292	} while (0)
1293	#else
1294	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1295	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1296	#endif
1297
1298	AssertPtr(pVM);
1299	AssertPtr(pVCpu);
1300	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1301	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1302	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1303
1304	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1305	AssertPtr(pCpu);
1306
1307	/*
1308	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1309	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1310	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1311	*/
1312	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1313	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1314	{
1315	++pCpu->uCurrentAsid;
1316	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1317	{
1318	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1319	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1320	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1321	}
1322
1323	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1324	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1325	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1326
1327	/*
1328	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1329	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1330	*/
1331	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1332	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1333	HMVMX_SET_TAGGED_TLB_FLUSHED();
1334	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1335	}
1336
1337	/* Check for explicit TLB shootdowns. */
1338	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1339	{
1340	/*
1341	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1342	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1343	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1344	* but not guest-physical mappings.
1345	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1346	*/
1347	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1348	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1349	HMVMX_SET_TAGGED_TLB_FLUSHED();
1350	}
1351
1352	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1353	* not be executed. See hmQueueInvlPage() where it is commented
1354	* out. Support individual entry flushing someday. */
1355	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1356	{
1357	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1358
1359	/*
1360	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1361	* as supported by the CPU.
1362	*/
1363	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1364	{
1365	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1366	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1367	}
1368	else
1369	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1370
1371	HMVMX_SET_TAGGED_TLB_FLUSHED();
1372	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1373	}
1374
1375	pVCpu->hm.s.TlbShootdown.cPages = 0;
1376	pVCpu->hm.s.fForceTLBFlush = false;
1377
1378	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1379
1380	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1381	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1382	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1383	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1384	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1385	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1386	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1387	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1388
1389	/* Update VMCS with the VPID. */
1390	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1391	AssertRC(rc);
1392
1393	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1394	}
1395
1396
1397	/**
1398	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1399	*
1400	* @returns VBox status code.
1401	* @param pVM Pointer to the VM.
1402	* @param pVCpu Pointer to the VMCPU.
1403	*
1404	* @remarks Called with interrupts disabled.
1405	*/
1406	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1407	{
1408	AssertPtr(pVM);
1409	AssertPtr(pVCpu);
1410	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1411	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1412
1413	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1414	AssertPtr(pCpu);
1415
1416	/*
1417	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1418	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1419	*/
1420	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1421	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1422	{
1423	pVCpu->hm.s.fForceTLBFlush = true;
1424	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1425	}
1426
1427	/* Check for explicit TLB shootdown flushes. */
1428	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1429	{
1430	pVCpu->hm.s.fForceTLBFlush = true;
1431	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1432	}
1433
1434	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1435	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1436
1437	if (pVCpu->hm.s.fForceTLBFlush)
1438	{
1439	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1440	pVCpu->hm.s.fForceTLBFlush = false;
1441	}
1442	else
1443	{
1444	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1445	* not be executed. See hmQueueInvlPage() where it is commented
1446	* out. Support individual entry flushing someday. */
1447	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1448	{
1449	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1450	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1451	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1452	}
1453	else
1454	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1455	}
1456
1457	pVCpu->hm.s.TlbShootdown.cPages = 0;
1458	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1459	}
1460
1461
1462	/**
1463	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1464	*
1465	* @returns VBox status code.
1466	* @param pVM Pointer to the VM.
1467	* @param pVCpu Pointer to the VMCPU.
1468	*
1469	* @remarks Called with interrupts disabled.
1470	*/
1471	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1472	{
1473	AssertPtr(pVM);
1474	AssertPtr(pVCpu);
1475	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1476	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1477
1478	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1479
1480	/*
1481	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1482	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1483	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1484	*/
1485	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1486	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1487	{
1488	pVCpu->hm.s.fForceTLBFlush = true;
1489	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1490	}
1491
1492	/* Check for explicit TLB shootdown flushes. */
1493	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1494	{
1495	/*
1496	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1497	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1498	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1499	*/
1500	pVCpu->hm.s.fForceTLBFlush = true;
1501	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1502	}
1503
1504	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1505	if (pVCpu->hm.s.fForceTLBFlush)
1506	{
1507	++pCpu->uCurrentAsid;
1508	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1509	{
1510	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1511	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1512	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1513	}
1514
1515	pVCpu->hm.s.fForceTLBFlush = false;
1516	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1517	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1518	if (pCpu->fFlushAsidBeforeUse)
1519	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1520	}
1521	else
1522	{
1523	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1524	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1525	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1526	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1527
1528	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1529	* not be executed. See hmQueueInvlPage() where it is commented
1530	* out. Support individual entry flushing someday. */
1531	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1532	{
1533	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1534	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1535	{
1536	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1537	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1538	}
1539	else
1540	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1541	}
1542	else
1543	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1544	}
1545
1546	pVCpu->hm.s.TlbShootdown.cPages = 0;
1547	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1548
1549	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1550	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1551	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1552	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1553	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1554	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1555
1556	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1557	AssertRC(rc);
1558	}
1559
1560
1561	/**
1562	* Flushes the guest TLB entry based on CPU capabilities.
1563	*
1564	* @param pVCpu Pointer to the VMCPU.
1565	*/
1566	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1567	{
1568	PVM pVM = pVCpu->CTX_SUFF(pVM);
1569	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1570	{
1571	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1572	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1573	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1574	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1575	default:
1576	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1577	break;
1578	}
1579	}
1580
1581
1582	/**
1583	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1584	* TLB entries from the host TLB before VM-entry.
1585	*
1586	* @returns VBox status code.
1587	* @param pVM Pointer to the VM.
1588	*/
1589	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1590	{
1591	/*
1592	* Determine optimal flush type for Nested Paging.
1593	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1594	* guest execution (see hmR3InitFinalizeR0()).
1595	*/
1596	if (pVM->hm.s.fNestedPaging)
1597	{
1598	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1599	{
1600	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1601	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1602	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1603	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1604	else
1605	{
1606	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1607	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1608	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1609	}
1610
1611	/* Make sure the write-back cacheable memory type for EPT is supported. */
1612	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1613	{
1614	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1615	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1616	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1617	}
1618	}
1619	else
1620	{
1621	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1622	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1623	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1624	}
1625	}
1626
1627	/*
1628	* Determine optimal flush type for VPID.
1629	*/
1630	if (pVM->hm.s.vmx.fVpid)
1631	{
1632	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1633	{
1634	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1635	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1636	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1637	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1638	else
1639	{
1640	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1641	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1642	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1643	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1644	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1645	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1646	pVM->hm.s.vmx.fVpid = false;
1647	}
1648	}
1649	else
1650	{
1651	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1652	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1653	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1654	pVM->hm.s.vmx.fVpid = false;
1655	}
1656	}
1657
1658	/*
1659	* Setup the handler for flushing tagged-TLBs.
1660	*/
1661	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1662	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1663	else if (pVM->hm.s.fNestedPaging)
1664	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1665	else if (pVM->hm.s.vmx.fVpid)
1666	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1667	else
1668	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1669	return VINF_SUCCESS;
1670	}
1671
1672
1673	/**
1674	* Sets up pin-based VM-execution controls in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVM Pointer to the VM.
1678	* @param pVCpu Pointer to the VMCPU.
1679	*/
1680	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1681	{
1682	AssertPtr(pVM);
1683	AssertPtr(pVCpu);
1684
1685	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1686	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1687
1688	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1689	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1690	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1691
1692	/* Enable the VMX preemption timer. */
1693	if (pVM->hm.s.vmx.fUsePreemptTimer)
1694	{
1695	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1696	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1697	}
1698
1699	if ((val & zap) != val)
1700	{
1701	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1702	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1703	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1704	}
1705
1706	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1707	AssertRCReturn(rc, rc);
1708
1709	/* Update VCPU with the currently set pin-based VM-execution controls. */
1710	pVCpu->hm.s.vmx.u32PinCtls = val;
1711	return rc;
1712	}
1713
1714
1715	/**
1716	* Sets up processor-based VM-execution controls in the VMCS.
1717	*
1718	* @returns VBox status code.
1719	* @param pVM Pointer to the VM.
1720	* @param pVMCPU Pointer to the VMCPU.
1721	*/
1722	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1723	{
1724	AssertPtr(pVM);
1725	AssertPtr(pVCpu);
1726
1727	int rc = VERR_INTERNAL_ERROR_5;
1728	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1729	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1730
1731	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1732	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1733	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1734	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1735	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1736	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1737	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1738
1739	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1740	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1741	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1742	{
1743	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1744	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1745	}
1746
1747	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1748	if (!pVM->hm.s.fNestedPaging)
1749	{
1750	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1751	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1752	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1753	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1754	}
1755
1756	/* Use TPR shadowing if supported by the CPU. */
1757	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1758	{
1759	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1760	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1761	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1762	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1763	AssertRCReturn(rc, rc);
1764
1765	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1766	/* CR8 writes causes a VM-exit based on TPR threshold. */
1767	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1768	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1769	}
1770	else
1771	{
1772	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1773	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1774	}
1775
1776	/* Use MSR-bitmaps if supported by the CPU. */
1777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1778	{
1779	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1780
1781	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1782	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1783	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1784	AssertRCReturn(rc, rc);
1785
1786	/*
1787	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1788	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1789	*/
1790	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1791	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1792	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1793	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1794	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1795	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1796	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1797	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1798	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1799	}
1800
1801	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1802	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1803	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1804
1805	if ((val & zap) != val)
1806	{
1807	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1808	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1809	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1810	}
1811
1812	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1813	AssertRCReturn(rc, rc);
1814
1815	/* Update VCPU with the currently set processor-based VM-execution controls. */
1816	pVCpu->hm.s.vmx.u32ProcCtls = val;
1817
1818	/*
1819	* Secondary processor-based VM-execution controls.
1820	*/
1821	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1822	{
1823	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1824	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1825
1826	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1827	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1828
1829	if (pVM->hm.s.fNestedPaging)
1830	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1831	else
1832	{
1833	/*
1834	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1835	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1836	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1837	*/
1838	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1839	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1840	}
1841
1842	if (pVM->hm.s.vmx.fVpid)
1843	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1844
1845	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1846	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1847
1848	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1849	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1850	* done dynamically. */
1851	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1852	{
1853	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1854	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1855	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1856	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1857	AssertRCReturn(rc, rc);
1858	}
1859
1860	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1861	{
1862	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1864	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1865	}
1866
1867	if ((val & zap) != val)
1868	{
1869	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1870	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1871	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1872	}
1873
1874	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1875	AssertRCReturn(rc, rc);
1876
1877	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1878	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1879	}
1880	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1881	{
1882	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1883	"available\n"));
1884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1885	}
1886
1887	return VINF_SUCCESS;
1888	}
1889
1890
1891	/**
1892	* Sets up miscellaneous (everything other than Pin & Processor-based
1893	* VM-execution) control fields in the VMCS.
1894	*
1895	* @returns VBox status code.
1896	* @param pVM Pointer to the VM.
1897	* @param pVCpu Pointer to the VMCPU.
1898	*/
1899	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1900	{
1901	AssertPtr(pVM);
1902	AssertPtr(pVCpu);
1903
1904	int rc = VERR_GENERAL_FAILURE;
1905
1906	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1907	#if 0
1908	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1909	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1910	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1911
1912	/*
1913	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1914	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1915	* We thus use the exception bitmap to control it rather than use both.
1916	*/
1917	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1918	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1919
1920	/** @todo Explore possibility of using IO-bitmaps. */
1921	/* All IO & IOIO instructions cause VM-exits. */
1922	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1923	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1924
1925	/* Initialize the MSR-bitmap area. */
1926	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1927	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1928	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1929	#endif
1930
1931	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1932	/* Setup MSR autoloading/storing. */
1933	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1934	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1935	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1936	AssertRCReturn(rc, rc);
1937	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1938	AssertRCReturn(rc, rc);
1939
1940	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1941	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1942	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1943	AssertRCReturn(rc, rc);
1944	#endif
1945
1946	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1947	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1948	AssertRCReturn(rc, rc);
1949
1950	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1951	#if 0
1952	/* Setup debug controls */
1953	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1954	AssertRCReturn(rc, rc);
1955	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1956	AssertRCReturn(rc, rc);
1957	#endif
1958
1959	return rc;
1960	}
1961
1962
1963	/**
1964	* Sets up the initial exception bitmap in the VMCS based on static conditions
1965	* (i.e. conditions that cannot ever change at runtime).
1966	*
1967	* @returns VBox status code.
1968	* @param pVM Pointer to the VM.
1969	* @param pVCpu Pointer to the VMCPU.
1970	*/
1971	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1972	{
1973	AssertPtr(pVM);
1974	AssertPtr(pVCpu);
1975
1976	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1977
1978	uint32_t u32XcptBitmap = 0;
1979
1980	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1981	if (!pVM->hm.s.fNestedPaging)
1982	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1983
1984	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1985	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1986	AssertRCReturn(rc, rc);
1987	return rc;
1988	}
1989
1990
1991	/**
1992	* Sets up the initial guest-state mask. The guest-state mask is consulted
1993	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1994	* for the nested virtualization case (as it would cause a VM-exit).
1995	*
1996	* @param pVCpu Pointer to the VMCPU.
1997	*/
1998	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1999	{
2000	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2001	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2002	return VINF_SUCCESS;
2003	}
2004
2005
2006	/**
2007	* Does per-VM VT-x initialization.
2008	*
2009	* @returns VBox status code.
2010	* @param pVM Pointer to the VM.
2011	*/
2012	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2013	{
2014	LogFlowFunc(("pVM=%p\n", pVM));
2015
2016	int rc = hmR0VmxStructsAlloc(pVM);
2017	if (RT_FAILURE(rc))
2018	{
2019	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2020	return rc;
2021	}
2022
2023	return VINF_SUCCESS;
2024	}
2025
2026
2027	/**
2028	* Does per-VM VT-x termination.
2029	*
2030	* @returns VBox status code.
2031	* @param pVM Pointer to the VM.
2032	*/
2033	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2034	{
2035	LogFlowFunc(("pVM=%p\n", pVM));
2036
2037	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2038	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2039	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2040	#endif
2041	hmR0VmxStructsFree(pVM);
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Sets up the VM for execution under VT-x.
2048	* This function is only called once per-VM during initialization.
2049	*
2050	* @returns VBox status code.
2051	* @param pVM Pointer to the VM.
2052	*/
2053	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2054	{
2055	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2056	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2057
2058	LogFlowFunc(("pVM=%p\n", pVM));
2059
2060	/*
2061	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2062	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2063	*/
2064	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2065	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2066	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2067	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2068	{
2069	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2070	return VERR_INTERNAL_ERROR;
2071	}
2072
2073	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2074	/*
2075	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2076	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2077	*/
2078	if ( pVM->hm.s.fAllow64BitGuests
2079	&& !HMVMX_IS_64BIT_HOST_MODE())
2080	{
2081	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2082	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2083	}
2084	#endif
2085
2086	/* Initialize these always, see hmR3InitFinalizeR0().*/
2087	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2088	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2089
2090	/* Setup the tagged-TLB flush handlers. */
2091	int rc = hmR0VmxSetupTaggedTlb(pVM);
2092	if (RT_FAILURE(rc))
2093	{
2094	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2095	return rc;
2096	}
2097
2098	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2099	{
2100	PVMCPU pVCpu = &pVM->aCpus[i];
2101	AssertPtr(pVCpu);
2102	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2103
2104	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2105	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2106
2107	/* Set revision dword at the beginning of the VMCS structure. */
2108	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2109
2110	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2111	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2112	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2113	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2114
2115	/* Load this VMCS as the current VMCS. */
2116	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2117	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2118	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2119
2120	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2121	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2122	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2123
2124	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2125	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2126	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2127
2128	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2129	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2130	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2131
2132	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2133	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2134	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2135
2136	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2137	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2138	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2139
2140	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2141	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2142	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2143	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2144	#endif
2145
2146	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2147	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2148	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2149	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2150
2151	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2152	}
2153
2154	return VINF_SUCCESS;
2155	}
2156
2157
2158	/**
2159	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2160	* the VMCS.
2161	*
2162	* @returns VBox status code.
2163	* @param pVM Pointer to the VM.
2164	* @param pVCpu Pointer to the VMCPU.
2165	*/
2166	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2167	{
2168	RTCCUINTREG uReg = ASMGetCR0();
2169	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2170	AssertRCReturn(rc, rc);
2171
2172	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2173	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2174	if (HMVMX_IS_64BIT_HOST_MODE())
2175	{
2176	uint64_t uRegCR3 = HMR0Get64bitCR3();
2177	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2178	}
2179	else
2180	#endif
2181	{
2182	uReg = ASMGetCR3();
2183	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2184	}
2185	AssertRCReturn(rc, rc);
2186
2187	uReg = ASMGetCR4();
2188	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2189	AssertRCReturn(rc, rc);
2190	return rc;
2191	}
2192
2193
2194	/**
2195	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2196	* the host-state area in the VMCS.
2197	*
2198	* @returns VBox status code.
2199	* @param pVM Pointer to the VM.
2200	* @param pVCpu Pointer to the VMCPU.
2201	*/
2202	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2203	{
2204	int rc = VERR_INTERNAL_ERROR_5;
2205	RTSEL uSelDS = 0;
2206	RTSEL uSelES = 0;
2207	RTSEL uSelFS = 0;
2208	RTSEL uSelGS = 0;
2209	RTSEL uSelTR = 0;
2210
2211	/*
2212	* Host DS, ES, FS and GS segment registers.
2213	*/
2214	#if HC_ARCH_BITS == 64
2215	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2216	uSelDS = ASMGetDS();
2217	uSelES = ASMGetES();
2218	uSelFS = ASMGetFS();
2219	uSelGS = ASMGetGS();
2220	#endif
2221
2222	/*
2223	* Host CS and SS segment registers.
2224	*/
2225	RTSEL uSelCS;
2226	RTSEL uSelSS;
2227	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2228	if (HMVMX_IS_64BIT_HOST_MODE())
2229	{
2230	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2231	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2232	}
2233	else
2234	{
2235	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2236	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2237	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2238	}
2239	#else
2240	uSelCS = ASMGetCS();
2241	uSelSS = ASMGetSS();
2242	#endif
2243
2244	/*
2245	* Host TR segment register.
2246	*/
2247	uSelTR = ASMGetTR();
2248
2249	#if HC_ARCH_BITS == 64
2250	/*
2251	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2252	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2253	*/
2254	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2255	{
2256	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2257	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2258	uSelDS = 0;
2259	}
2260	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2261	{
2262	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2263	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2264	uSelES = 0;
2265	}
2266	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2267	{
2268	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2269	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2270	uSelFS = 0;
2271	}
2272	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2273	{
2274	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2275	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2276	uSelGS = 0;
2277	}
2278	#endif
2279
2280	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2281	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2282	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2283	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2284	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2285	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2286	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2287	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2288	Assert(uSelCS);
2289	Assert(uSelTR);
2290
2291	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2292	#if 0
2293	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2294	Assert(uSelSS != 0);
2295	#endif
2296
2297	/* Write these host selector fields into the host-state area in the VMCS. */
2298	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2299	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2300	#if HC_ARCH_BITS == 64
2301	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2302	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2303	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2304	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2305	#endif
2306	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2307
2308	/*
2309	* Host GDTR and IDTR.
2310	*/
2311	RTGDTR Gdtr;
2312	RT_ZERO(Gdtr);
2313	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2314	if (HMVMX_IS_64BIT_HOST_MODE())
2315	{
2316	X86XDTR64 Gdtr64;
2317	X86XDTR64 Idtr64;
2318	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2319	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2320	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2321
2322	Gdtr.cbGdt = Gdtr64.cb;
2323	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2324	}
2325	else
2326	#endif
2327	{
2328	RTIDTR Idtr;
2329	ASMGetGDTR(&Gdtr);
2330	ASMGetIDTR(&Idtr);
2331	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2332	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2333
2334	#if HC_ARCH_BITS == 64
2335	/*
2336	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2337	* maximum limit (0xffff) on every VM-exit.
2338	*/
2339	if (Gdtr.cbGdt != 0xffff)
2340	{
2341	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2342	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2343	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2344	}
2345
2346	/*
2347	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2348	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2349	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2350	*/
2351	if (Idtr.cbIdt < 0x0fff)
2352	{
2353	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2354	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2355	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2356	}
2357	#endif
2358	}
2359
2360	/*
2361	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2362	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2363	*/
2364	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2365	{
2366	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2367	return VERR_VMX_INVALID_HOST_STATE;
2368	}
2369
2370	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2371	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2372	if (HMVMX_IS_64BIT_HOST_MODE())
2373	{
2374	/* We need the 64-bit TR base for hybrid darwin. */
2375	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2376	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2377	}
2378	else
2379	#endif
2380	{
2381	uintptr_t uTRBase;
2382	#if HC_ARCH_BITS == 64
2383	uTRBase = X86DESC64_BASE(pDesc);
2384
2385	/*
2386	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2387	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2388	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2389	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2390	*
2391	* [1] See Intel spec. 3.5 "System Descriptor Types".
2392	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2393	*/
2394	Assert(pDesc->System.u4Type == 11);
2395	if ( pDesc->System.u16LimitLow != 0x67
2396	\|\| pDesc->System.u4LimitHigh)
2397	{
2398	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2399	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2400
2401	/* Store the GDTR here as we need it while restoring TR. */
2402	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2403	}
2404	#else
2405	uTRBase = X86DESC_BASE(pDesc);
2406	#endif
2407	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2408	}
2409	AssertRCReturn(rc, rc);
2410
2411	/*
2412	* Host FS base and GS base.
2413	*/
2414	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2415	if (HMVMX_IS_64BIT_HOST_MODE())
2416	{
2417	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2418	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2419	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2420	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2421
2422	# if HC_ARCH_BITS == 64
2423	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2424	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2425	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2426	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2427	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2428	# endif
2429	}
2430	#endif
2431	return rc;
2432	}
2433
2434
2435	/**
2436	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2437	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2438	* the host after every successful VM exit.
2439	*
2440	* @returns VBox status code.
2441	* @param pVM Pointer to the VM.
2442	* @param pVCpu Pointer to the VMCPU.
2443	*/
2444	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2445	{
2446	AssertPtr(pVCpu);
2447	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2448
2449	int rc = VINF_SUCCESS;
2450	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2451	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2452	uint32_t cHostMsrs = 0;
2453	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2454
2455	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2456	{
2457	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2458
2459	# if HC_ARCH_BITS == 64
2460	/* Paranoia. 64-bit code requires these bits to be set always. */
2461	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2462
2463	/*
2464	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2465	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2466	* some reason (e.g. allow transparent reads) we would activate the code below.
2467	*/
2468	# if 0
2469	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2470	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2471	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2472	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2473	if (CPUMIsGuestInLongMode(pVCpu))
2474	{
2475	uint64_t u64GuestEfer;
2476	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2477	AssertRC(rc);
2478
2479	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2480	{
2481	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2482	pHostMsr->u32Reserved = 0;
2483	pHostMsr->u64Value = u64HostEfer;
2484	pHostMsr++; cHostMsrs++;
2485	}
2486	}
2487	# endif
2488	# else /* HC_ARCH_BITS != 64 */
2489	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2490	pHostMsr->u32Reserved = 0;
2491	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2492	if (CPUMIsGuestInLongMode(pVCpu))
2493	{
2494	/* Must match the EFER value in our 64 bits switcher. */
2495	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2496	}
2497	else
2498	# endif
2499	pHostMsr->u64Value = u64HostEfer;
2500	pHostMsr++; cHostMsrs++;
2501	# endif /* HC_ARCH_BITS == 64 */
2502	}
2503
2504	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2505	if (HMVMX_IS_64BIT_HOST_MODE())
2506	{
2507	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2508	pHostMsr->u32Reserved = 0;
2509	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2510	pHostMsr++; cHostMsrs++;
2511	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2512	pHostMsr->u32Reserved = 0;
2513	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2514	pHostMsr++; cHostMsrs++;
2515	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2516	pHostMsr->u32Reserved = 0;
2517	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2518	pHostMsr++; cHostMsrs++;
2519	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2520	pHostMsr->u32Reserved = 0;
2521	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2522	pHostMsr++; cHostMsrs++;
2523	}
2524	# endif
2525
2526	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2527	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2528	{
2529	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2530	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2531	}
2532
2533	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2534	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2535
2536	/*
2537	* Host Sysenter MSRs.
2538	*/
2539	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2540	AssertRCReturn(rc, rc);
2541	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2542	if (HMVMX_IS_64BIT_HOST_MODE())
2543	{
2544	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2545	AssertRCReturn(rc, rc);
2546	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2547	}
2548	else
2549	{
2550	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2551	AssertRCReturn(rc, rc);
2552	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2553	}
2554	#elif HC_ARCH_BITS == 32
2555	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2556	AssertRCReturn(rc, rc);
2557	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2558	#else
2559	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2560	AssertRCReturn(rc, rc);
2561	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2562	#endif
2563	AssertRCReturn(rc, rc);
2564
2565	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2566	* hmR0VmxSetupExitCtls() !! */
2567	return rc;
2568	}
2569
2570
2571	/**
2572	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2573	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2574	* controls".
2575	*
2576	* @returns VBox status code.
2577	* @param pVCpu Pointer to the VMCPU.
2578	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2579	* out-of-sync. Make sure to update the required fields
2580	* before using them.
2581	*
2582	* @remarks No-long-jump zone!!!
2583	*/
2584	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2585	{
2586	int rc = VINF_SUCCESS;
2587	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2588	{
2589	PVM pVM = pVCpu->CTX_SUFF(pVM);
2590	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2591	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2592
2593	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2594	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2595
2596	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2597	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2598	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2599	else
2600	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2601
2602	/*
2603	* The following should not be set (since we're not in SMM mode):
2604	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2605	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2606	*/
2607
2608	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2609	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2610	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2611
2612	if ((val & zap) != val)
2613	{
2614	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2615	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2616	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2617	}
2618
2619	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2620	AssertRCReturn(rc, rc);
2621
2622	/* Update VCPU with the currently set VM-exit controls. */
2623	pVCpu->hm.s.vmx.u32EntryCtls = val;
2624	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2625	}
2626	return rc;
2627	}
2628
2629
2630	/**
2631	* Sets up the VM-exit controls in the VMCS.
2632	*
2633	* @returns VBox status code.
2634	* @param pVM Pointer to the VM.
2635	* @param pVCpu Pointer to the VMCPU.
2636	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2637	* out-of-sync. Make sure to update the required fields
2638	* before using them.
2639	*
2640	* @remarks requires EFER.
2641	*/
2642	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2643	{
2644	int rc = VINF_SUCCESS;
2645	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2646	{
2647	PVM pVM = pVCpu->CTX_SUFF(pVM);
2648	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2649	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2650
2651	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2652	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2653
2654	/*
2655	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2656	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2657	*/
2658	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2659	if (HMVMX_IS_64BIT_HOST_MODE())
2660	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2661	else
2662	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2663	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2664	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2665	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2666	else
2667	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2668	#endif
2669
2670	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2671	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2672
2673	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2674	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2675	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2676	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2677	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2678
2679	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2680	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2681
2682	if ((val & zap) != val)
2683	{
2684	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2685	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2686	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2687	}
2688
2689	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2690	AssertRCReturn(rc, rc);
2691
2692	/* Update VCPU with the currently set VM-exit controls. */
2693	pVCpu->hm.s.vmx.u32ExitCtls = val;
2694	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2695	}
2696	return rc;
2697	}
2698
2699
2700	/**
2701	* Loads the guest APIC and related state.
2702	*
2703	* @returns VBox status code.
2704	* @param pVM Pointer to the VM.
2705	* @param pVCpu Pointer to the VMCPU.
2706	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2707	* out-of-sync. Make sure to update the required fields
2708	* before using them.
2709	*/
2710	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2711	{
2712	int rc = VINF_SUCCESS;
2713	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2714	{
2715	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2716	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2717	{
2718	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2719
2720	bool fPendingIntr = false;
2721	uint8_t u8Tpr = 0;
2722	uint8_t u8PendingIntr = 0;
2723	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2724	AssertRCReturn(rc, rc);
2725
2726	/*
2727	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2728	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2729	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2730	* the interrupt when we VM-exit for other reasons.
2731	*/
2732	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2733	uint32_t u32TprThreshold = 0;
2734	if (fPendingIntr)
2735	{
2736	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2737	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2738	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2739	if (u8PendingPriority <= u8TprPriority)
2740	u32TprThreshold = u8PendingPriority;
2741	else
2742	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2743	}
2744	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2745
2746	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2747	AssertRCReturn(rc, rc);
2748	}
2749
2750	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2751	}
2752	return rc;
2753	}
2754
2755
2756	/**
2757	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2758	*
2759	* @returns Guest's interruptibility-state.
2760	* @param pVCpu Pointer to the VMCPU.
2761	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2762	* out-of-sync. Make sure to update the required fields
2763	* before using them.
2764	*
2765	* @remarks No-long-jump zone!!!
2766	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2767	*/
2768	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2769	{
2770	/*
2771	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2772	* inhibit interrupts or clear any existing interrupt-inhibition.
2773	*/
2774	uint32_t uIntrState = 0;
2775	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2776	{
2777	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2778	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2779	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2780	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2781	{
2782	/*
2783	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2784	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2785	*/
2786	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2787	}
2788	else if (pMixedCtx->eflags.Bits.u1IF)
2789	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2790	else
2791	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2792	}
2793	return uIntrState;
2794	}
2795
2796
2797	/**
2798	* Loads the guest's interruptibility-state into the guest-state area in the
2799	* VMCS.
2800	*
2801	* @returns VBox status code.
2802	* @param pVCpu Pointer to the VMCPU.
2803	* @param uIntrState The interruptibility-state to set.
2804	*/
2805	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2806	{
2807	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2808	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2809	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2810	AssertRCReturn(rc, rc);
2811	return rc;
2812	}
2813
2814
2815	/**
2816	* Loads the guest's RIP into the guest-state area in the VMCS.
2817	*
2818	* @returns VBox status code.
2819	* @param pVCpu Pointer to the VMCPU.
2820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2821	* out-of-sync. Make sure to update the required fields
2822	* before using them.
2823	*
2824	* @remarks No-long-jump zone!!!
2825	*/
2826	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2827	{
2828	int rc = VINF_SUCCESS;
2829	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2830	{
2831	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2832	AssertRCReturn(rc, rc);
2833	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2834	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2835	}
2836	return rc;
2837	}
2838
2839
2840	/**
2841	* Loads the guest's RSP into the guest-state area in the VMCS.
2842	*
2843	* @returns VBox status code.
2844	* @param pVCpu Pointer to the VMCPU.
2845	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2846	* out-of-sync. Make sure to update the required fields
2847	* before using them.
2848	*
2849	* @remarks No-long-jump zone!!!
2850	*/
2851	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2852	{
2853	int rc = VINF_SUCCESS;
2854	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2855	{
2856	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2857	AssertRCReturn(rc, rc);
2858	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2859	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2860	}
2861	return rc;
2862	}
2863
2864
2865	/**
2866	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2867	*
2868	* @returns VBox status code.
2869	* @param pVCpu Pointer to the VMCPU.
2870	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2871	* out-of-sync. Make sure to update the required fields
2872	* before using them.
2873	*
2874	* @remarks No-long-jump zone!!!
2875	*/
2876	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2877	{
2878	int rc = VINF_SUCCESS;
2879	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2880	{
2881	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2882	Let us assert it as such and use 32-bit VMWRITE. */
2883	Assert(!(pMixedCtx->rflags.u64 >> 32));
2884	X86EFLAGS Eflags = pMixedCtx->eflags;
2885	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2886	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2887
2888	/*
2889	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2890	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2891	*/
2892	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2893	{
2894	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2895	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2896	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2897	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2898	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2899	}
2900
2901	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2902	AssertRCReturn(rc, rc);
2903
2904	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2905	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2906	}
2907	return rc;
2908	}
2909
2910
2911	/**
2912	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2913	*
2914	* @returns VBox status code.
2915	* @param pVCpu Pointer to the VMCPU.
2916	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2917	* out-of-sync. Make sure to update the required fields
2918	* before using them.
2919	*
2920	* @remarks No-long-jump zone!!!
2921	*/
2922	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2923	{
2924	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2925	AssertRCReturn(rc, rc);
2926	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2927	AssertRCReturn(rc, rc);
2928	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2929	AssertRCReturn(rc, rc);
2930	return rc;
2931	}
2932
2933
2934	/**
2935	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2936	* in the VMCS.
2937	*
2938	* @returns VBox status code.
2939	* @param pVM Pointer to the VM.
2940	* @param pVCpu Pointer to the VMCPU.
2941	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2942	* out-of-sync. Make sure to update the required fields
2943	* before using them.
2944	*
2945	* @remarks No-long-jump zone!!!
2946	*/
2947	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2948	{
2949	int rc = VINF_SUCCESS;
2950	PVM pVM = pVCpu->CTX_SUFF(pVM);
2951
2952	/*
2953	* Guest CR0.
2954	* Guest FPU.
2955	*/
2956	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2957	{
2958	Assert(!(pCtx->cr0 >> 32));
2959	uint32_t u32GuestCR0 = pCtx->cr0;
2960
2961	/* The guest's view (read access) of its CR0 is unblemished. */
2962	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2963	AssertRCReturn(rc, rc);
2964	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2965
2966	/* Setup VT-x's view of the guest CR0. */
2967	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2968	if (pVM->hm.s.fNestedPaging)
2969	{
2970	if (CPUMIsGuestPagingEnabledEx(pCtx))
2971	{
2972	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2973	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2974	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2975	}
2976	else
2977	{
2978	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2979	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2980	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2981	}
2982
2983	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2984	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2985	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2986
2987	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2988	AssertRCReturn(rc, rc);
2989	}
2990	else
2991	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2992
2993	/*
2994	* Guest FPU bits.
2995	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2996	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2997	*/
2998	u32GuestCR0 \|= X86_CR0_NE;
2999	bool fInterceptNM = false;
3000	if (CPUMIsGuestFPUStateActive(pVCpu))
3001	{
3002	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3003	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3004	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3005	}
3006	else
3007	{
3008	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3009	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3010	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3011	}
3012
3013	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3014	bool fInterceptMF = false;
3015	if (!(pCtx->cr0 & X86_CR0_NE))
3016	fInterceptMF = true;
3017
3018	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3019	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3020	{
3021	Assert(PDMVmmDevHeapIsEnabled(pVM));
3022	Assert(pVM->hm.s.vmx.pRealModeTSS);
3023	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3024	fInterceptNM = true;
3025	fInterceptMF = true;
3026	}
3027	else
3028	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3029
3030	if (fInterceptNM)
3031	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3032	else
3033	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3034
3035	if (fInterceptMF)
3036	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3037	else
3038	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3039
3040	/* Additional intercepts for debugging, define these yourself explicitly. */
3041	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3042	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3043	\| RT_BIT(X86_XCPT_BP)
3044	\| RT_BIT(X86_XCPT_DB)
3045	\| RT_BIT(X86_XCPT_DE)
3046	\| RT_BIT(X86_XCPT_NM)
3047	\| RT_BIT(X86_XCPT_UD)
3048	\| RT_BIT(X86_XCPT_NP)
3049	\| RT_BIT(X86_XCPT_SS)
3050	\| RT_BIT(X86_XCPT_GP)
3051	\| RT_BIT(X86_XCPT_PF)
3052	\| RT_BIT(X86_XCPT_MF)
3053	;
3054	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3055	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3056	#endif
3057
3058	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3059
3060	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3061	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3062	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3063	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3064	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3065	else
3066	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3067
3068	u32GuestCR0 \|= uSetCR0;
3069	u32GuestCR0 &= uZapCR0;
3070	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3071
3072	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3073	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3074	AssertRCReturn(rc, rc);
3075	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3076	AssertRCReturn(rc, rc);
3077	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3078
3079	/*
3080	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3081	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3082	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3083	*/
3084	uint32_t u32CR0Mask = 0;
3085	u32CR0Mask = X86_CR0_PE
3086	\| X86_CR0_NE
3087	\| X86_CR0_WP
3088	\| X86_CR0_PG
3089	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3090	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3091	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3092	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3093	u32CR0Mask &= ~X86_CR0_PE;
3094	if (pVM->hm.s.fNestedPaging)
3095	u32CR0Mask &= ~X86_CR0_WP;
3096
3097	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3098	if (fInterceptNM)
3099	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3100	else
3101	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3102
3103	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3104	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3105	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3106	AssertRCReturn(rc, rc);
3107
3108	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3109	}
3110
3111	/*
3112	* Guest CR2.
3113	* It's always loaded in the assembler code. Nothing to do here.
3114	*/
3115
3116	/*
3117	* Guest CR3.
3118	*/
3119	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3120	{
3121	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3122	if (pVM->hm.s.fNestedPaging)
3123	{
3124	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3125
3126	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3127	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3128	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3129	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3130
3131	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3132	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3133	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3134
3135	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3136	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3137	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3138	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3139
3140	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3141	AssertRCReturn(rc, rc);
3142	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3143
3144	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3145	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3146	{
3147	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3148	if (CPUMIsGuestInPAEModeEx(pCtx))
3149	{
3150	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3151	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3152	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3153	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3154	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3155	}
3156
3157	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3158	have Unrestricted Execution to handle the guest when it's not using paging. */
3159	GCPhysGuestCR3 = pCtx->cr3;
3160	}
3161	else
3162	{
3163	/*
3164	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3165	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3166	* EPT takes care of translating it to host-physical addresses.
3167	*/
3168	RTGCPHYS GCPhys;
3169	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3170	Assert(PDMVmmDevHeapIsEnabled(pVM));
3171
3172	/* We obtain it here every time as the guest could have relocated this PCI region. */
3173	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3174	AssertRCReturn(rc, rc);
3175
3176	GCPhysGuestCR3 = GCPhys;
3177	}
3178
3179	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3180	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3181	}
3182	else
3183	{
3184	/* Non-nested paging case, just use the hypervisor's CR3. */
3185	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3186
3187	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3188	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3189	}
3190	AssertRCReturn(rc, rc);
3191
3192	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3193	}
3194
3195	/*
3196	* Guest CR4.
3197	*/
3198	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3199	{
3200	Assert(!(pCtx->cr4 >> 32));
3201	uint32_t u32GuestCR4 = pCtx->cr4;
3202
3203	/* The guest's view of its CR4 is unblemished. */
3204	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3205	AssertRCReturn(rc, rc);
3206	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3207
3208	/* Setup VT-x's view of the guest CR4. */
3209	/*
3210	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3211	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3212	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3213	*/
3214	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3215	{
3216	Assert(pVM->hm.s.vmx.pRealModeTSS);
3217	Assert(PDMVmmDevHeapIsEnabled(pVM));
3218	u32GuestCR4 &= ~X86_CR4_VME;
3219	}
3220
3221	if (pVM->hm.s.fNestedPaging)
3222	{
3223	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3224	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3225	{
3226	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3227	u32GuestCR4 \|= X86_CR4_PSE;
3228	/* Our identity mapping is a 32 bits page directory. */
3229	u32GuestCR4 &= ~X86_CR4_PAE;
3230	}
3231	/* else use guest CR4.*/
3232	}
3233	else
3234	{
3235	/*
3236	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3237	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3238	*/
3239	switch (pVCpu->hm.s.enmShadowMode)
3240	{
3241	case PGMMODE_REAL: /* Real-mode. */
3242	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3243	case PGMMODE_32_BIT: /* 32-bit paging. */
3244	{
3245	u32GuestCR4 &= ~X86_CR4_PAE;
3246	break;
3247	}
3248
3249	case PGMMODE_PAE: /* PAE paging. */
3250	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3251	{
3252	u32GuestCR4 \|= X86_CR4_PAE;
3253	break;
3254	}
3255
3256	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3257	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3258	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3259	break;
3260	#endif
3261	default:
3262	AssertFailed();
3263	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3264	}
3265	}
3266
3267	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3268	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3269	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3270	u32GuestCR4 \|= uSetCR4;
3271	u32GuestCR4 &= uZapCR4;
3272
3273	/* Write VT-x's view of the guest CR4 into the VMCS. */
3274	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3275	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3276	AssertRCReturn(rc, rc);
3277
3278	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3279	uint32_t u32CR4Mask = 0;
3280	u32CR4Mask = X86_CR4_VME
3281	\| X86_CR4_PAE
3282	\| X86_CR4_PGE
3283	\| X86_CR4_PSE
3284	\| X86_CR4_VMXE;
3285	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3286	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3287	AssertRCReturn(rc, rc);
3288
3289	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3290	}
3291	return rc;
3292	}
3293
3294
3295	/**
3296	* Loads the guest debug registers into the guest-state area in the VMCS.
3297	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3298	*
3299	* @returns VBox status code.
3300	* @param pVCpu Pointer to the VMCPU.
3301	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3302	* out-of-sync. Make sure to update the required fields
3303	* before using them.
3304	*
3305	* @remarks No-long-jump zone!!!
3306	*/
3307	static int hmR0VmxLoadGuestDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3308	{
3309	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3310	return VINF_SUCCESS;
3311
3312	#ifdef VBOX_STRICT
3313	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3314	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3315	{
3316	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3317	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3318	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3319	}
3320	#endif
3321
3322	int rc;
3323	PVM pVM = pVCpu->CTX_SUFF(pVM);
3324	bool fInterceptDB = false;
3325	bool fInterceptMovDRx = false;
3326	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3327	{
3328	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3329	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3330	{
3331	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3332	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3333	AssertRCReturn(rc, rc);
3334	Assert(fInterceptDB == false);
3335	}
3336	else
3337	{
3338	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3339	pVCpu->hm.s.fClearTrapFlag = true;
3340	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3341	fInterceptDB = true;
3342	}
3343	}
3344
3345	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3346	{
3347	/*
3348	* Use the combined guest and host DRx values found in the hypervisor
3349	* register set because the debugger has breakpoints active or someone
3350	* is single stepping on the host side without a monitor trap flag.
3351	*
3352	* Note! DBGF expects a clean DR6 state before executing guest code.
3353	*/
3354	if (!CPUMIsHyperDebugStateActive(pVCpu))
3355	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3356	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3357	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3358
3359	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3360	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3361	AssertRCReturn(rc, rc);
3362
3363	fInterceptDB = true;
3364	fInterceptMovDRx = true;
3365	}
3366	else
3367	{
3368	/*
3369	* If the guest has enabled debug registers, we need to load them prior to
3370	* executing guest code so they'll trigger at the right time.
3371	*/
3372	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3373	{
3374	if (!CPUMIsGuestDebugStateActive(pVCpu))
3375	{
3376	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3377	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3378	}
3379	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3380	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3381	}
3382	/*
3383	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3384	* must intercept #DB in order to maintain a correct DR6 guest value.
3385	*/
3386	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3387	{
3388	fInterceptMovDRx = true;
3389	fInterceptDB = true;
3390	}
3391
3392	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3393	AssertRCReturn(rc, rc);
3394	}
3395
3396	/*
3397	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3398	*/
3399	if (fInterceptDB)
3400	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3401	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3402	{
3403	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3404	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3405	#endif
3406	}
3407	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3408	AssertRCReturn(rc, rc);
3409
3410	/*
3411	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3412	*/
3413	if (fInterceptMovDRx)
3414	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3415	else
3416	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3417	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3418	AssertRCReturn(rc, rc);
3419
3420	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3421	return VINF_SUCCESS;
3422	}
3423
3424
3425	#ifdef VBOX_STRICT
3426	/**
3427	* Strict function to validate segment registers.
3428	*
3429	* @remarks Requires CR0.
3430	*/
3431	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3432	{
3433	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3434	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3435	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3436	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3437	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3438	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3439	{
3440	/* Protected mode checks */
3441	/* CS */
3442	Assert(pCtx->cs.Attr.n.u1Present);
3443	Assert(!(pCtx->cs.Attr.u & 0xf00));
3444	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3445	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3446	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3447	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3448	\|\| (pCtx->cs.Attr.n.u1Granularity));
3449	/* CS cannot be loaded with NULL in protected mode. */
3450	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3451	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3452	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3453	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3454	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3455	else
3456	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3457	/* SS */
3458	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3459	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3460	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3461	if ( !(pCtx->cr0 & X86_CR0_PE)
3462	\|\| pCtx->cs.Attr.n.u4Type == 3)
3463	{
3464	Assert(!pCtx->ss.Attr.n.u2Dpl);
3465	}
3466	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3467	{
3468	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3469	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3470	Assert(pCtx->ss.Attr.n.u1Present);
3471	Assert(!(pCtx->ss.Attr.u & 0xf00));
3472	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3473	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3474	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3475	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3476	\|\| (pCtx->ss.Attr.n.u1Granularity));
3477	}
3478	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3479	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3480	{
3481	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3482	Assert(pCtx->ds.Attr.n.u1Present);
3483	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3484	Assert(!(pCtx->ds.Attr.u & 0xf00));
3485	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3486	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3487	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3488	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3489	\|\| (pCtx->ds.Attr.n.u1Granularity));
3490	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3491	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3492	}
3493	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3494	{
3495	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3496	Assert(pCtx->es.Attr.n.u1Present);
3497	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3498	Assert(!(pCtx->es.Attr.u & 0xf00));
3499	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3500	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3501	\|\| !(pCtx->es.Attr.n.u1Granularity));
3502	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3503	\|\| (pCtx->es.Attr.n.u1Granularity));
3504	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3505	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3506	}
3507	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3508	{
3509	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3510	Assert(pCtx->fs.Attr.n.u1Present);
3511	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3512	Assert(!(pCtx->fs.Attr.u & 0xf00));
3513	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3514	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3515	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3516	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3517	\|\| (pCtx->fs.Attr.n.u1Granularity));
3518	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3519	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3520	}
3521	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3522	{
3523	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3524	Assert(pCtx->gs.Attr.n.u1Present);
3525	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3526	Assert(!(pCtx->gs.Attr.u & 0xf00));
3527	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3528	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3529	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3530	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3531	\|\| (pCtx->gs.Attr.n.u1Granularity));
3532	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3533	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3534	}
3535	/* 64-bit capable CPUs. */
3536	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3537	Assert(!(pCtx->cs.u64Base >> 32));
3538	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3539	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3540	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3541	# endif
3542	}
3543	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3544	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3545	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3546	{
3547	/* Real and v86 mode checks. */
3548	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3549	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3550	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3551	{
3552	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3553	}
3554	else
3555	{
3556	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3557	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3558	}
3559
3560	/* CS */
3561	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3562	Assert(pCtx->cs.u32Limit == 0xffff);
3563	Assert(u32CSAttr == 0xf3);
3564	/* SS */
3565	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3566	Assert(pCtx->ss.u32Limit == 0xffff);
3567	Assert(u32SSAttr == 0xf3);
3568	/* DS */
3569	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3570	Assert(pCtx->ds.u32Limit == 0xffff);
3571	Assert(u32DSAttr == 0xf3);
3572	/* ES */
3573	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3574	Assert(pCtx->es.u32Limit == 0xffff);
3575	Assert(u32ESAttr == 0xf3);
3576	/* FS */
3577	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3578	Assert(pCtx->fs.u32Limit == 0xffff);
3579	Assert(u32FSAttr == 0xf3);
3580	/* GS */
3581	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3582	Assert(pCtx->gs.u32Limit == 0xffff);
3583	Assert(u32GSAttr == 0xf3);
3584	/* 64-bit capable CPUs. */
3585	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3586	Assert(!(pCtx->cs.u64Base >> 32));
3587	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3588	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3589	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3590	# endif
3591	}
3592	}
3593	#endif /* VBOX_STRICT */
3594
3595
3596	/**
3597	* Writes a guest segment register into the guest-state area in the VMCS.
3598	*
3599	* @returns VBox status code.
3600	* @param pVCpu Pointer to the VMCPU.
3601	* @param idxSel Index of the selector in the VMCS.
3602	* @param idxLimit Index of the segment limit in the VMCS.
3603	* @param idxBase Index of the segment base in the VMCS.
3604	* @param idxAccess Index of the access rights of the segment in the VMCS.
3605	* @param pSelReg Pointer to the segment selector.
3606	* @param pCtx Pointer to the guest-CPU context.
3607	*
3608	* @remarks No-long-jump zone!!!
3609	*/
3610	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3611	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3612	{
3613	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3614	AssertRCReturn(rc, rc);
3615	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3616	AssertRCReturn(rc, rc);
3617	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3618	AssertRCReturn(rc, rc);
3619
3620	uint32_t u32Access = pSelReg->Attr.u;
3621	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3622	{
3623	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3624	u32Access = 0xf3;
3625	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3626	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3627	}
3628	else
3629	{
3630	/*
3631	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3632	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3633	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3634	* loaded in protected-mode have their attribute as 0.
3635	*/
3636	if (!u32Access)
3637	u32Access = X86DESCATTR_UNUSABLE;
3638	}
3639
3640	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3641	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3642	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3643
3644	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3645	AssertRCReturn(rc, rc);
3646	return rc;
3647	}
3648
3649
3650	/**
3651	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3652	* into the guest-state area in the VMCS.
3653	*
3654	* @returns VBox status code.
3655	* @param pVM Pointer to the VM.
3656	* @param pVCPU Pointer to the VMCPU.
3657	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3658	* out-of-sync. Make sure to update the required fields
3659	* before using them.
3660	*
3661	* @remarks Requires CR0 (strict builds validation).
3662	* @remarks No-long-jump zone!!!
3663	*/
3664	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3665	{
3666	int rc = VERR_INTERNAL_ERROR_5;
3667	PVM pVM = pVCpu->CTX_SUFF(pVM);
3668
3669	/*
3670	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3671	*/
3672	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3673	{
3674	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3675	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3676	{
3677	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3678	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3679	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3680	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3681	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3682	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3683	}
3684
3685	#ifdef VBOX_WITH_REM
3686	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3687	{
3688	Assert(pVM->hm.s.vmx.pRealModeTSS);
3689	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3690	if ( pVCpu->hm.s.vmx.fWasInRealMode
3691	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3692	{
3693	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3694	in real-mode (e.g. OpenBSD 4.0) */
3695	REMFlushTBs(pVM);
3696	Log4(("Load: Switch to protected mode detected!\n"));
3697	pVCpu->hm.s.vmx.fWasInRealMode = false;
3698	}
3699	}
3700	#endif
3701	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3702	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3703	AssertRCReturn(rc, rc);
3704	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3705	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3706	AssertRCReturn(rc, rc);
3707	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3708	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3709	AssertRCReturn(rc, rc);
3710	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3711	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3712	AssertRCReturn(rc, rc);
3713	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3714	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3715	AssertRCReturn(rc, rc);
3716	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3717	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3718	AssertRCReturn(rc, rc);
3719
3720	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3721	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3722	#ifdef VBOX_STRICT
3723	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3724	#endif
3725	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3726	}
3727
3728	/*
3729	* Guest TR.
3730	*/
3731	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3732	{
3733	/*
3734	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3735	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3736	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3737	*/
3738	uint16_t u16Sel = 0;
3739	uint32_t u32Limit = 0;
3740	uint64_t u64Base = 0;
3741	uint32_t u32AccessRights = 0;
3742
3743	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3744	{
3745	u16Sel = pMixedCtx->tr.Sel;
3746	u32Limit = pMixedCtx->tr.u32Limit;
3747	u64Base = pMixedCtx->tr.u64Base;
3748	u32AccessRights = pMixedCtx->tr.Attr.u;
3749	}
3750	else
3751	{
3752	Assert(pVM->hm.s.vmx.pRealModeTSS);
3753	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3754
3755	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3756	RTGCPHYS GCPhys;
3757	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3758	AssertRCReturn(rc, rc);
3759
3760	X86DESCATTR DescAttr;
3761	DescAttr.u = 0;
3762	DescAttr.n.u1Present = 1;
3763	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3764
3765	u16Sel = 0;
3766	u32Limit = HM_VTX_TSS_SIZE;
3767	u64Base = GCPhys; /* in real-mode phys = virt. */
3768	u32AccessRights = DescAttr.u;
3769	}
3770
3771	/* Validate. */
3772	Assert(!(u16Sel & RT_BIT(2)));
3773	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3774	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3775	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3776	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3777	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3778	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3779	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3780	Assert( (u32Limit & 0xfff) == 0xfff
3781	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3782	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3783	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3784
3785	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3786	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3787	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3788	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3789
3790	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3791	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3792	}
3793
3794	/*
3795	* Guest GDTR.
3796	*/
3797	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3798	{
3799	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3800	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3801
3802	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3803	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3804	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3805	}
3806
3807	/*
3808	* Guest LDTR.
3809	*/
3810	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3811	{
3812	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3813	uint32_t u32Access = 0;
3814	if (!pMixedCtx->ldtr.Attr.u)
3815	u32Access = X86DESCATTR_UNUSABLE;
3816	else
3817	u32Access = pMixedCtx->ldtr.Attr.u;
3818
3819	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3820	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3821	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3822	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3823
3824	/* Validate. */
3825	if (!(u32Access & X86DESCATTR_UNUSABLE))
3826	{
3827	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3828	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3829	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3830	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3831	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3832	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3833	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3834	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3835	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3836	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3837	}
3838
3839	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3840	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3841	}
3842
3843	/*
3844	* Guest IDTR.
3845	*/
3846	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3847	{
3848	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3849	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3850
3851	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3852	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3853	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3854	}
3855
3856	return VINF_SUCCESS;
3857	}
3858
3859
3860	/**
3861	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3862	* areas. These MSRs will automatically be loaded to the host CPU on every
3863	* successful VM entry and stored from the host CPU on every successful VM exit.
3864	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3865	*
3866	* @returns VBox status code.
3867	* @param pVCpu Pointer to the VMCPU.
3868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3869	* out-of-sync. Make sure to update the required fields
3870	* before using them.
3871	*
3872	* @remarks No-long-jump zone!!!
3873	*/
3874	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3875	{
3876	AssertPtr(pVCpu);
3877	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3878
3879	/*
3880	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3881	*/
3882	int rc = VINF_SUCCESS;
3883	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3884	{
3885	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3886	PVM pVM = pVCpu->CTX_SUFF(pVM);
3887	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3888	uint32_t cGuestMsrs = 0;
3889
3890	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3891	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3892	* when the guest really is in 64-bit mode. */
3893	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3894	if (fSupportsLongMode)
3895	{
3896	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3897	pGuestMsr->u32Reserved = 0;
3898	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3899	pGuestMsr++; cGuestMsrs++;
3900	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3901	pGuestMsr->u32Reserved = 0;
3902	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3903	pGuestMsr++; cGuestMsrs++;
3904	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3905	pGuestMsr->u32Reserved = 0;
3906	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3907	pGuestMsr++; cGuestMsrs++;
3908	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3909	pGuestMsr->u32Reserved = 0;
3910	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3911	pGuestMsr++; cGuestMsrs++;
3912	}
3913
3914	/*
3915	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3916	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3917	*/
3918	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3919	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3920	{
3921	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3922	pGuestMsr->u32Reserved = 0;
3923	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3924	AssertRCReturn(rc, rc);
3925	pGuestMsr++; cGuestMsrs++;
3926	}
3927
3928	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3929	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3930	{
3931	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3932	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3933	}
3934
3935	/* Update the VCPU's copy of the guest MSR count. */
3936	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3937	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3938	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3939	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3940
3941	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3942	}
3943
3944	/*
3945	* Guest Sysenter MSRs.
3946	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3947	* VM-exits on WRMSRs for these MSRs.
3948	*/
3949	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3950	{
3951	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3952	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3953	}
3954	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3955	{
3956	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3957	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3958	}
3959	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3960	{
3961	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3962	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3963	}
3964
3965	return rc;
3966	}
3967
3968
3969	/**
3970	* Loads the guest activity state into the guest-state area in the VMCS.
3971	*
3972	* @returns VBox status code.
3973	* @param pVCpu Pointer to the VMCPU.
3974	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3975	* out-of-sync. Make sure to update the required fields
3976	* before using them.
3977	*
3978	* @remarks No-long-jump zone!!!
3979	*/
3980	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3981	{
3982	/** @todo See if we can make use of other states, e.g.
3983	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3984	int rc = VINF_SUCCESS;
3985	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3986	{
3987	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3988	AssertRCReturn(rc, rc);
3989	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3990	}
3991	return rc;
3992	}
3993
3994
3995	/**
3996	* Sets up the appropriate function to run guest code.
3997	*
3998	* @returns VBox status code.
3999	* @param pVCpu Pointer to the VMCPU.
4000	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4001	* out-of-sync. Make sure to update the required fields
4002	* before using them.
4003	*
4004	* @remarks No-long-jump zone!!!
4005	*/
4006	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4007	{
4008	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4009	{
4010	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4011	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4012	#endif
4013	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4014	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4015	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4016	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4017	#else
4018	/* 64-bit host or hybrid host. */
4019	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4020	#endif
4021	}
4022	else
4023	{
4024	/* Guest is not in long mode, use the 32-bit handler. */
4025	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4026	}
4027	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4028	return VINF_SUCCESS;
4029	}
4030
4031
4032	/**
4033	* Wrapper for running the guest code in VT-x.
4034	*
4035	* @returns VBox strict status code.
4036	* @param pVM Pointer to the VM.
4037	* @param pVCpu Pointer to the VMCPU.
4038	* @param pCtx Pointer to the guest-CPU context.
4039	*
4040	* @remarks No-long-jump zone!!!
4041	*/
4042	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4043	{
4044	/*
4045	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4046	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4047	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4048	*/
4049	#ifdef VBOX_WITH_KERNEL_USING_XMM
4050	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4051	#else
4052	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4053	#endif
4054	}
4055
4056
4057	/**
4058	* Reports world-switch error and dumps some useful debug info.
4059	*
4060	* @param pVM Pointer to the VM.
4061	* @param pVCpu Pointer to the VMCPU.
4062	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4063	* @param pCtx Pointer to the guest-CPU context.
4064	* @param pVmxTransient Pointer to the VMX transient structure (only
4065	* exitReason updated).
4066	*/
4067	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4068	{
4069	Assert(pVM);
4070	Assert(pVCpu);
4071	Assert(pCtx);
4072	Assert(pVmxTransient);
4073	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4074
4075	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4076	switch (rcVMRun)
4077	{
4078	case VERR_VMX_INVALID_VMXON_PTR:
4079	AssertFailed();
4080	break;
4081	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4082	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4083	{
4084	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4085	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4086	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4087	AssertRC(rc);
4088
4089	#ifdef VBOX_STRICT
4090	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4091	pVmxTransient->uExitReason));
4092	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4093	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4094	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4095	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4096	else
4097	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4098
4099	/* VMX control bits. */
4100	uint32_t u32Val;
4101	uint64_t u64Val;
4102	HMVMXHCUINTREG uHCReg;
4103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4104	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4106	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4108	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4110	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4112	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4114	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4116	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4118	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4120	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4121	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4122	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4123	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4124	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4125	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4126	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4128	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4129	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4130	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4131	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4132	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4133	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4134	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4135	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4136	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4137	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4138	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4139	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4140	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4141	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4142	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4143	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4144	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4145
4146	/* Guest bits. */
4147	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4148	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4149	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4150	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4151	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4152	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4153	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4154	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4155
4156	/* Host bits. */
4157	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4158	Log4(("Host CR0 %#RHr\n", uHCReg));
4159	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4160	Log4(("Host CR3 %#RHr\n", uHCReg));
4161	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4162	Log4(("Host CR4 %#RHr\n", uHCReg));
4163
4164	RTGDTR HostGdtr;
4165	PCX86DESCHC pDesc;
4166	ASMGetGDTR(&HostGdtr);
4167	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4168	Log4(("Host CS %#08x\n", u32Val));
4169	if (u32Val < HostGdtr.cbGdt)
4170	{
4171	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4172	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4173	}
4174
4175	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4176	Log4(("Host DS %#08x\n", u32Val));
4177	if (u32Val < HostGdtr.cbGdt)
4178	{
4179	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4180	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4181	}
4182
4183	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4184	Log4(("Host ES %#08x\n", u32Val));
4185	if (u32Val < HostGdtr.cbGdt)
4186	{
4187	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4188	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4189	}
4190
4191	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4192	Log4(("Host FS %#08x\n", u32Val));
4193	if (u32Val < HostGdtr.cbGdt)
4194	{
4195	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4196	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4197	}
4198
4199	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4200	Log4(("Host GS %#08x\n", u32Val));
4201	if (u32Val < HostGdtr.cbGdt)
4202	{
4203	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4204	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4205	}
4206
4207	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4208	Log4(("Host SS %#08x\n", u32Val));
4209	if (u32Val < HostGdtr.cbGdt)
4210	{
4211	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4212	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4213	}
4214
4215	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4216	Log4(("Host TR %#08x\n", u32Val));
4217	if (u32Val < HostGdtr.cbGdt)
4218	{
4219	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4220	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4221	}
4222
4223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4224	Log4(("Host TR Base %#RHv\n", uHCReg));
4225	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4226	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4227	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4228	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4229	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4230	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4231	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4232	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4233	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4234	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4235	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4236	Log4(("Host RSP %#RHv\n", uHCReg));
4237	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4238	Log4(("Host RIP %#RHv\n", uHCReg));
4239	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4240	if (HMVMX_IS_64BIT_HOST_MODE())
4241	{
4242	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4243	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4244	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4245	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4246	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4247	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4248	}
4249	# endif
4250	#endif /* VBOX_STRICT */
4251	break;
4252	}
4253
4254	default:
4255	/* Impossible */
4256	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4257	break;
4258	}
4259	NOREF(pVM);
4260	}
4261
4262
4263	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4264	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4265	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4266	#endif
4267	#ifdef VBOX_STRICT
4268	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4269	{
4270	switch (idxField)
4271	{
4272	case VMX_VMCS_GUEST_RIP:
4273	case VMX_VMCS_GUEST_RSP:
4274	case VMX_VMCS_GUEST_SYSENTER_EIP:
4275	case VMX_VMCS_GUEST_SYSENTER_ESP:
4276	case VMX_VMCS_GUEST_GDTR_BASE:
4277	case VMX_VMCS_GUEST_IDTR_BASE:
4278	case VMX_VMCS_GUEST_CS_BASE:
4279	case VMX_VMCS_GUEST_DS_BASE:
4280	case VMX_VMCS_GUEST_ES_BASE:
4281	case VMX_VMCS_GUEST_FS_BASE:
4282	case VMX_VMCS_GUEST_GS_BASE:
4283	case VMX_VMCS_GUEST_SS_BASE:
4284	case VMX_VMCS_GUEST_LDTR_BASE:
4285	case VMX_VMCS_GUEST_TR_BASE:
4286	case VMX_VMCS_GUEST_CR3:
4287	return true;
4288	}
4289	return false;
4290	}
4291
4292	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4293	{
4294	switch (idxField)
4295	{
4296	/* Read-only fields. */
4297	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4298	return true;
4299	}
4300	/* Remaining readable fields should also be writable. */
4301	return hmR0VmxIsValidWriteField(idxField);
4302	}
4303	#endif /* VBOX_STRICT */
4304
4305
4306	/**
4307	* Executes the specified handler in 64-bit mode.
4308	*
4309	* @returns VBox status code.
4310	* @param pVM Pointer to the VM.
4311	* @param pVCpu Pointer to the VMCPU.
4312	* @param pCtx Pointer to the guest CPU context.
4313	* @param enmOp The operation to perform.
4314	* @param cbParam Number of parameters.
4315	* @param paParam Array of 32-bit parameters.
4316	*/
4317	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4318	uint32_t *paParam)
4319	{
4320	int rc, rc2;
4321	PHMGLOBALCPUINFO pCpu;
4322	RTHCPHYS HCPhysCpuPage;
4323	RTCCUINTREG uOldEflags;
4324
4325	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4326	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4327	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4328	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4329
4330	#ifdef VBOX_STRICT
4331	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4332	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4333
4334	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4335	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4336	#endif
4337
4338	/* Disable interrupts. */
4339	uOldEflags = ASMIntDisableFlags();
4340
4341	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4342	RTCPUID idHostCpu = RTMpCpuId();
4343	CPUMR0SetLApic(pVM, idHostCpu);
4344	#endif
4345
4346	pCpu = HMR0GetCurrentCpu();
4347	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4348
4349	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4350	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4351
4352	/* Leave VMX Root Mode. */
4353	VMXDisable();
4354
4355	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4356
4357	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4358	CPUMSetHyperEIP(pVCpu, enmOp);
4359	for (int i = (int)cbParam - 1; i >= 0; i--)
4360	CPUMPushHyper(pVCpu, paParam[i]);
4361
4362	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4363
4364	/* Call the switcher. */
4365	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4366	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4367
4368	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4369	/* Make sure the VMX instructions don't cause #UD faults. */
4370	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4371
4372	/* Re-enter VMX Root Mode */
4373	rc2 = VMXEnable(HCPhysCpuPage);
4374	if (RT_FAILURE(rc2))
4375	{
4376	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4377	ASMSetFlags(uOldEflags);
4378	return rc2;
4379	}
4380
4381	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4382	AssertRC(rc2);
4383	Assert(!(ASMGetFlags() & X86_EFL_IF));
4384	ASMSetFlags(uOldEflags);
4385	return rc;
4386	}
4387
4388
4389	/**
4390	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4391	* supporting 64-bit guests.
4392	*
4393	* @returns VBox status code.
4394	* @param fResume Whether to VMLAUNCH or VMRESUME.
4395	* @param pCtx Pointer to the guest-CPU context.
4396	* @param pCache Pointer to the VMCS cache.
4397	* @param pVM Pointer to the VM.
4398	* @param pVCpu Pointer to the VMCPU.
4399	*/
4400	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4401	{
4402	uint32_t aParam[6];
4403	PHMGLOBALCPUINFO pCpu = NULL;
4404	RTHCPHYS HCPhysCpuPage = 0;
4405	int rc = VERR_INTERNAL_ERROR_5;
4406
4407	pCpu = HMR0GetCurrentCpu();
4408	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4409
4410	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4411	pCache->uPos = 1;
4412	pCache->interPD = PGMGetInterPaeCR3(pVM);
4413	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4414	#endif
4415
4416	#ifdef VBOX_STRICT
4417	pCache->TestIn.HCPhysCpuPage = 0;
4418	pCache->TestIn.HCPhysVmcs = 0;
4419	pCache->TestIn.pCache = 0;
4420	pCache->TestOut.HCPhysVmcs = 0;
4421	pCache->TestOut.pCache = 0;
4422	pCache->TestOut.pCtx = 0;
4423	pCache->TestOut.eflags = 0;
4424	#endif
4425
4426	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4427	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4428	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4429	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4430	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4431	aParam[5] = 0;
4432
4433	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4434	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4435	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4436	#endif
4437	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4438
4439	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4440	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4441	Assert(pCtx->dr[4] == 10);
4442	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4443	#endif
4444
4445	#ifdef VBOX_STRICT
4446	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4447	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4448	pVCpu->hm.s.vmx.HCPhysVmcs));
4449	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4450	pCache->TestOut.HCPhysVmcs));
4451	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4452	pCache->TestOut.pCache));
4453	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4454	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4455	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4456	pCache->TestOut.pCtx));
4457	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4458	#endif
4459	return rc;
4460	}
4461
4462
4463	/**
4464	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4465	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4466	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4467	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4468	*
4469	* @returns VBox status code.
4470	* @param pVM Pointer to the VM.
4471	* @param pVCpu Pointer to the VMCPU.
4472	*/
4473	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4474	{
4475	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4476	{ \
4477	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4478	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4479	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4480	++cReadFields; \
4481	}
4482
4483	AssertPtr(pVM);
4484	AssertPtr(pVCpu);
4485	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4486	uint32_t cReadFields = 0;
4487
4488	/*
4489	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4490	* and serve to indicate exceptions to the rules.
4491	*/
4492
4493	/* Guest-natural selector base fields. */
4494	#if 0
4495	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4498	#endif
4499	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4500	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4501	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4503	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4511	#if 0
4512	/* Unused natural width guest-state fields. */
4513	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4515	#endif
4516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4518
4519	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4520	#if 0
4521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4523	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4530	#endif
4531
4532	/* Natural width guest-state fields. */
4533	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4534	#if 0
4535	/* Currently unused field. */
4536	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4537	#endif
4538
4539	if (pVM->hm.s.fNestedPaging)
4540	{
4541	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4542	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4543	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4544	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4545	}
4546	else
4547	{
4548	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4549	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4550	}
4551
4552	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4553	return VINF_SUCCESS;
4554	}
4555
4556
4557	/**
4558	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4559	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4560	* darwin, running 64-bit guests).
4561	*
4562	* @returns VBox status code.
4563	* @param pVCpu Pointer to the VMCPU.
4564	* @param idxField The VMCS field encoding.
4565	* @param u64Val 16, 32 or 64 bits value.
4566	*/
4567	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4568	{
4569	int rc;
4570	switch (idxField)
4571	{
4572	/*
4573	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4574	*/
4575	/* 64-bit Control fields. */
4576	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4577	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4578	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4579	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4580	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4581	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4582	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4583	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4584	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4585	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4586	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4587	case VMX_VMCS64_CTRL_EPTP_FULL:
4588	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4589	/* 64-bit Guest-state fields. */
4590	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4591	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4592	case VMX_VMCS64_GUEST_PAT_FULL:
4593	case VMX_VMCS64_GUEST_EFER_FULL:
4594	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4595	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4596	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4597	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4598	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4599	/* 64-bit Host-state fields. */
4600	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4601	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4602	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4603	{
4604	rc = VMXWriteVmcs32(idxField, u64Val);
4605	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4606	break;
4607	}
4608
4609	/*
4610	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4611	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4612	*/
4613	/* Natural-width Guest-state fields. */
4614	case VMX_VMCS_GUEST_CR3:
4615	case VMX_VMCS_GUEST_ES_BASE:
4616	case VMX_VMCS_GUEST_CS_BASE:
4617	case VMX_VMCS_GUEST_SS_BASE:
4618	case VMX_VMCS_GUEST_DS_BASE:
4619	case VMX_VMCS_GUEST_FS_BASE:
4620	case VMX_VMCS_GUEST_GS_BASE:
4621	case VMX_VMCS_GUEST_LDTR_BASE:
4622	case VMX_VMCS_GUEST_TR_BASE:
4623	case VMX_VMCS_GUEST_GDTR_BASE:
4624	case VMX_VMCS_GUEST_IDTR_BASE:
4625	case VMX_VMCS_GUEST_RSP:
4626	case VMX_VMCS_GUEST_RIP:
4627	case VMX_VMCS_GUEST_SYSENTER_ESP:
4628	case VMX_VMCS_GUEST_SYSENTER_EIP:
4629	{
4630	if (!(u64Val >> 32))
4631	{
4632	/* If this field is 64-bit, VT-x will zero out the top bits. */
4633	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4634	}
4635	else
4636	{
4637	/* Assert that only the 32->64 switcher case should ever come here. */
4638	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4639	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4640	}
4641	break;
4642	}
4643
4644	default:
4645	{
4646	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4647	rc = VERR_INVALID_PARAMETER;
4648	break;
4649	}
4650	}
4651	AssertRCReturn(rc, rc);
4652	return rc;
4653	}
4654
4655
4656	/**
4657	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4658	* hosts (except darwin) for 64-bit guests.
4659	*
4660	* @param pVCpu Pointer to the VMCPU.
4661	* @param idxField The VMCS field encoding.
4662	* @param u64Val 16, 32 or 64 bits value.
4663	*/
4664	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4665	{
4666	AssertPtr(pVCpu);
4667	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4668
4669	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4670	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4671
4672	/* Make sure there are no duplicates. */
4673	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4674	{
4675	if (pCache->Write.aField[i] == idxField)
4676	{
4677	pCache->Write.aFieldVal[i] = u64Val;
4678	return VINF_SUCCESS;
4679	}
4680	}
4681
4682	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4683	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4684	pCache->Write.cValidEntries++;
4685	return VINF_SUCCESS;
4686	}
4687
4688	/* Enable later when the assembly code uses these as callbacks. */
4689	#if 0
4690	/*
4691	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4692	*
4693	* @param pVCpu Pointer to the VMCPU.
4694	* @param pCache Pointer to the VMCS cache.
4695	*
4696	* @remarks No-long-jump zone!!!
4697	*/
4698	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4699	{
4700	AssertPtr(pCache);
4701	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4702	{
4703	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4704	AssertRC(rc);
4705	}
4706	pCache->Write.cValidEntries = 0;
4707	}
4708
4709
4710	/**
4711	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4712	*
4713	* @param pVCpu Pointer to the VMCPU.
4714	* @param pCache Pointer to the VMCS cache.
4715	*
4716	* @remarks No-long-jump zone!!!
4717	*/
4718	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4719	{
4720	AssertPtr(pCache);
4721	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4722	{
4723	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4724	AssertRC(rc);
4725	}
4726	}
4727	#endif
4728	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4729
4730
4731	/**
4732	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4733	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4734	* timer.
4735	*
4736	* @returns VBox status code.
4737	* @param pVCpu Pointer to the VMCPU.
4738	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4739	* out-of-sync. Make sure to update the required fields
4740	* before using them.
4741	* @remarks No-long-jump zone!!!
4742	*/
4743	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4744	{
4745	int rc = VERR_INTERNAL_ERROR_5;
4746	bool fOffsettedTsc = false;
4747	PVM pVM = pVCpu->CTX_SUFF(pVM);
4748	if (pVM->hm.s.vmx.fUsePreemptTimer)
4749	{
4750	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4751
4752	/* Make sure the returned values have sane upper and lower boundaries. */
4753	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4754	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4755	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4756	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4757
4758	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4759	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4760	}
4761	else
4762	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4763
4764	if (fOffsettedTsc)
4765	{
4766	uint64_t u64CurTSC = ASMReadTSC();
4767	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4768	{
4769	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4770	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4771
4772	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4773	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4774	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4775	}
4776	else
4777	{
4778	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4779	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4780	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4781	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4782	}
4783	}
4784	else
4785	{
4786	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4787	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4788	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4789	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4790	}
4791	}
4792
4793
4794	/**
4795	* Determines if an exception is a contributory exception. Contributory
4796	* exceptions are ones which can cause double-faults. Page-fault is
4797	* intentionally not included here as it's a conditional contributory exception.
4798	*
4799	* @returns true if the exception is contributory, false otherwise.
4800	* @param uVector The exception vector.
4801	*/
4802	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4803	{
4804	switch (uVector)
4805	{
4806	case X86_XCPT_GP:
4807	case X86_XCPT_SS:
4808	case X86_XCPT_NP:
4809	case X86_XCPT_TS:
4810	case X86_XCPT_DE:
4811	return true;
4812	default:
4813	break;
4814	}
4815	return false;
4816	}
4817
4818
4819	/**
4820	* Sets an event as a pending event to be injected into the guest.
4821	*
4822	* @param pVCpu Pointer to the VMCPU.
4823	* @param u32IntrInfo The VM-entry interruption-information field.
4824	* @param cbInstr The VM-entry instruction length in bytes (for software
4825	* interrupts, exceptions and privileged software
4826	* exceptions).
4827	* @param u32ErrCode The VM-entry exception error code.
4828	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4829	* page-fault.
4830	*
4831	* @remarks Statistics counter assumes this is a guest event being injected or
4832	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4833	* always incremented.
4834	*/
4835	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4836	RTGCUINTPTR GCPtrFaultAddress)
4837	{
4838	Assert(!pVCpu->hm.s.Event.fPending);
4839	pVCpu->hm.s.Event.fPending = true;
4840	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4841	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4842	pVCpu->hm.s.Event.cbInstr = cbInstr;
4843	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4844
4845	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4846	}
4847
4848
4849	/**
4850	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4851	*
4852	* @param pVCpu Pointer to the VMCPU.
4853	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4854	* out-of-sync. Make sure to update the required fields
4855	* before using them.
4856	*/
4857	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4858	{
4859	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4860	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4861	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4862	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4863	}
4864
4865
4866	/**
4867	* Handle a condition that occurred while delivering an event through the guest
4868	* IDT.
4869	*
4870	* @returns VBox status code (informational error codes included).
4871	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4872	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4873	* continue execution of the guest which will delivery the #DF.
4874	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4875	*
4876	* @param pVCpu Pointer to the VMCPU.
4877	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4878	* out-of-sync. Make sure to update the required fields
4879	* before using them.
4880	* @param pVmxTransient Pointer to the VMX transient structure.
4881	*
4882	* @remarks No-long-jump zone!!!
4883	*/
4884	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4885	{
4886	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4887	AssertRC(rc);
4888	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4889	{
4890	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4891	AssertRCReturn(rc, rc);
4892
4893	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4894	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4895	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4896
4897	typedef enum
4898	{
4899	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4900	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4901	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4902	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4903	} VMXREFLECTXCPT;
4904
4905	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4906	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4907	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4908	{
4909	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4910	{
4911	enmReflect = VMXREFLECTXCPT_XCPT;
4912	#ifdef VBOX_STRICT
4913	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4914	&& uExitVector == X86_XCPT_PF)
4915	{
4916	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4917	}
4918	#endif
4919	if ( uExitVector == X86_XCPT_PF
4920	&& uIdtVector == X86_XCPT_PF)
4921	{
4922	pVmxTransient->fVectoringPF = true;
4923	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4924	}
4925	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4926	&& hmR0VmxIsContributoryXcpt(uExitVector)
4927	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4928	\|\| uIdtVector == X86_XCPT_PF))
4929	{
4930	enmReflect = VMXREFLECTXCPT_DF;
4931	}
4932	else if (uIdtVector == X86_XCPT_DF)
4933	enmReflect = VMXREFLECTXCPT_TF;
4934	}
4935	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4936	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4937	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4938	{
4939	/*
4940	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4941	* (whatever they are) as they reoccur when restarting the instruction.
4942	*/
4943	enmReflect = VMXREFLECTXCPT_XCPT;
4944	}
4945	}
4946	else
4947	{
4948	/*
4949	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4950	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4951	* original exception to the guest after handling the VM-exit.
4952	*/
4953	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4954	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4955	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4956	{
4957	enmReflect = VMXREFLECTXCPT_XCPT;
4958	}
4959	}
4960
4961	switch (enmReflect)
4962	{
4963	case VMXREFLECTXCPT_XCPT:
4964	{
4965	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4966	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4967	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
4968
4969	uint32_t u32ErrCode = 0;
4970	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
4971	{
4972	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4973	AssertRCReturn(rc, rc);
4974	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4975	}
4976
4977	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4978	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4979	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4980	rc = VINF_SUCCESS;
4981	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
4982	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4983
4984	break;
4985	}
4986
4987	case VMXREFLECTXCPT_DF:
4988	{
4989	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4990	rc = VINF_HM_DOUBLE_FAULT;
4991	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
4992	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4993
4994	break;
4995	}
4996
4997	case VMXREFLECTXCPT_TF:
4998	{
4999	rc = VINF_EM_RESET;
5000	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5001	uExitVector));
5002	break;
5003	}
5004
5005	default:
5006	Assert(rc == VINF_SUCCESS);
5007	break;
5008	}
5009	}
5010	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5011	return rc;
5012	}
5013
5014
5015	/**
5016	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5017	*
5018	* @returns VBox status code.
5019	* @param pVCpu Pointer to the VMCPU.
5020	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5021	* out-of-sync. Make sure to update the required fields
5022	* before using them.
5023	*
5024	* @remarks No-long-jump zone!!!
5025	*/
5026	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5027	{
5028	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5029	{
5030	uint32_t uVal = 0;
5031	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5032	AssertRCReturn(rc, rc);
5033	uint32_t uShadow = 0;
5034	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5035	AssertRCReturn(rc, rc);
5036
5037	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5038	CPUMSetGuestCR0(pVCpu, uVal);
5039	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5040	}
5041	return VINF_SUCCESS;
5042	}
5043
5044
5045	/**
5046	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5047	*
5048	* @returns VBox status code.
5049	* @param pVCpu Pointer to the VMCPU.
5050	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5051	* out-of-sync. Make sure to update the required fields
5052	* before using them.
5053	*
5054	* @remarks No-long-jump zone!!!
5055	*/
5056	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5057	{
5058	int rc = VINF_SUCCESS;
5059	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5060	{
5061	uint32_t uVal = 0;
5062	uint32_t uShadow = 0;
5063	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5064	AssertRCReturn(rc, rc);
5065	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5066	AssertRCReturn(rc, rc);
5067
5068	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5069	CPUMSetGuestCR4(pVCpu, uVal);
5070	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5071	}
5072	return rc;
5073	}
5074
5075
5076	/**
5077	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5078	*
5079	* @returns VBox status code.
5080	* @param pVCpu Pointer to the VMCPU.
5081	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5082	* out-of-sync. Make sure to update the required fields
5083	* before using them.
5084	*
5085	* @remarks No-long-jump zone!!!
5086	*/
5087	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5088	{
5089	int rc = VINF_SUCCESS;
5090	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5091	{
5092	uint64_t u64Val = 0;
5093	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5094	AssertRCReturn(rc, rc);
5095
5096	pMixedCtx->rip = u64Val;
5097	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5098	}
5099	return rc;
5100	}
5101
5102
5103	/**
5104	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5105	*
5106	* @returns VBox status code.
5107	* @param pVCpu Pointer to the VMCPU.
5108	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5109	* out-of-sync. Make sure to update the required fields
5110	* before using them.
5111	*
5112	* @remarks No-long-jump zone!!!
5113	*/
5114	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5115	{
5116	int rc = VINF_SUCCESS;
5117	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5118	{
5119	uint64_t u64Val = 0;
5120	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5121	AssertRCReturn(rc, rc);
5122
5123	pMixedCtx->rsp = u64Val;
5124	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5125	}
5126	return rc;
5127	}
5128
5129
5130	/**
5131	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5132	*
5133	* @returns VBox status code.
5134	* @param pVCpu Pointer to the VMCPU.
5135	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5136	* out-of-sync. Make sure to update the required fields
5137	* before using them.
5138	*
5139	* @remarks No-long-jump zone!!!
5140	*/
5141	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5142	{
5143	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5144	{
5145	uint32_t uVal = 0;
5146	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5147	AssertRCReturn(rc, rc);
5148
5149	pMixedCtx->eflags.u32 = uVal;
5150	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5151	{
5152	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5153	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5154
5155	pMixedCtx->eflags.Bits.u1VM = 0;
5156	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5157	}
5158
5159	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5160	}
5161	return VINF_SUCCESS;
5162	}
5163
5164
5165	/**
5166	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5167	* guest-CPU context.
5168	*/
5169	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5170	{
5171	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5172	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5173	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5174	return rc;
5175	}
5176
5177
5178	/**
5179	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5180	* from the guest-state area in the VMCS.
5181	*
5182	* @param pVCpu Pointer to the VMCPU.
5183	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5184	* out-of-sync. Make sure to update the required fields
5185	* before using them.
5186	*
5187	* @remarks No-long-jump zone!!!
5188	*/
5189	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5190	{
5191	uint32_t uIntrState = 0;
5192	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5193	AssertRC(rc);
5194
5195	if (!uIntrState)
5196	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5197	else
5198	{
5199	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5200	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5201	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5202	AssertRC(rc);
5203	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5204	AssertRC(rc);
5205
5206	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5207	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5208	}
5209	}
5210
5211
5212	/**
5213	* Saves the guest's activity state.
5214	*
5215	* @returns VBox status code.
5216	* @param pVCpu Pointer to the VMCPU.
5217	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5218	* out-of-sync. Make sure to update the required fields
5219	* before using them.
5220	*
5221	* @remarks No-long-jump zone!!!
5222	*/
5223	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5224	{
5225	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5226	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5227	return VINF_SUCCESS;
5228	}
5229
5230
5231	/**
5232	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5233	* the current VMCS into the guest-CPU context.
5234	*
5235	* @returns VBox status code.
5236	* @param pVCpu Pointer to the VMCPU.
5237	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5238	* out-of-sync. Make sure to update the required fields
5239	* before using them.
5240	*
5241	* @remarks No-long-jump zone!!!
5242	*/
5243	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5244	{
5245	int rc = VINF_SUCCESS;
5246	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5247	{
5248	uint32_t u32Val = 0;
5249	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5250	pMixedCtx->SysEnter.cs = u32Val;
5251	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5252	}
5253
5254	uint64_t u64Val = 0;
5255	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5256	{
5257	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5258	pMixedCtx->SysEnter.eip = u64Val;
5259	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5260	}
5261	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5262	{
5263	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5264	pMixedCtx->SysEnter.esp = u64Val;
5265	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5266	}
5267	return rc;
5268	}
5269
5270
5271	/**
5272	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5273	* context.
5274	*
5275	* @returns VBox status code.
5276	* @param pVCpu Pointer to the VMCPU.
5277	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5278	* out-of-sync. Make sure to update the required fields
5279	* before using them.
5280	*
5281	* @remarks No-long-jump zone!!!
5282	*/
5283	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5284	{
5285	int rc = VINF_SUCCESS;
5286	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5287	{
5288	uint64_t u64Val = 0;
5289	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5290	pMixedCtx->fs.u64Base = u64Val;
5291	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5292	}
5293	return rc;
5294	}
5295
5296
5297	/**
5298	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5299	* context.
5300	*
5301	* @returns VBox status code.
5302	* @param pVCpu Pointer to the VMCPU.
5303	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5304	* out-of-sync. Make sure to update the required fields
5305	* before using them.
5306	*
5307	* @remarks No-long-jump zone!!!
5308	*/
5309	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5310	{
5311	int rc = VINF_SUCCESS;
5312	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5313	{
5314	uint64_t u64Val = 0;
5315	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5316	pMixedCtx->gs.u64Base = u64Val;
5317	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5318	}
5319	return rc;
5320	}
5321
5322
5323	/**
5324	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5325	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5326	* and TSC_AUX.
5327	*
5328	* @returns VBox status code.
5329	* @param pVCpu Pointer to the VMCPU.
5330	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5331	* out-of-sync. Make sure to update the required fields
5332	* before using them.
5333	*
5334	* @remarks No-long-jump zone!!!
5335	*/
5336	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5337	{
5338	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5339	return VINF_SUCCESS;
5340
5341	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5342	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5343	{
5344	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5345	pMsr += i;
5346	switch (pMsr->u32IndexMSR)
5347	{
5348	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5349	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5350	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5351	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5352	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5353	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5354	default:
5355	{
5356	AssertFailed();
5357	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5358	}
5359	}
5360	}
5361	#endif
5362
5363	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5364	return VINF_SUCCESS;
5365	}
5366
5367
5368	/**
5369	* Saves the guest control registers from the current VMCS into the guest-CPU
5370	* context.
5371	*
5372	* @returns VBox status code.
5373	* @param pVCpu Pointer to the VMCPU.
5374	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5375	* out-of-sync. Make sure to update the required fields
5376	* before using them.
5377	*
5378	* @remarks No-long-jump zone!!!
5379	*/
5380	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5381	{
5382	/* Guest CR0. Guest FPU. */
5383	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5384	AssertRCReturn(rc, rc);
5385
5386	/* Guest CR4. */
5387	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5388	AssertRCReturn(rc, rc);
5389
5390	/* Guest CR2 - updated always during the world-switch or in #PF. */
5391	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5392	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5393	{
5394	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5395	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5396
5397	PVM pVM = pVCpu->CTX_SUFF(pVM);
5398	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5399	\|\| ( pVM->hm.s.fNestedPaging
5400	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5401	{
5402	uint64_t u64Val = 0;
5403	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5404	if (pMixedCtx->cr3 != u64Val)
5405	{
5406	CPUMSetGuestCR3(pVCpu, u64Val);
5407	if (VMMRZCallRing3IsEnabled(pVCpu))
5408	{
5409	PGMUpdateCR3(pVCpu, u64Val);
5410	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5411	}
5412	else
5413	{
5414	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5415	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5416	}
5417	}
5418
5419	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5420	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5421	{
5422	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5423	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5424	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5425	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5426
5427	if (VMMRZCallRing3IsEnabled(pVCpu))
5428	{
5429	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5430	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5431	}
5432	else
5433	{
5434	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5435	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5436	}
5437	}
5438	}
5439
5440	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5441	}
5442
5443	/*
5444	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5445	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5446	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5447	*
5448	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5449	*/
5450	if (VMMRZCallRing3IsEnabled(pVCpu))
5451	{
5452	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5453	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5454
5455	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5456	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5457
5458	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5459	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5460	}
5461
5462	return rc;
5463	}
5464
5465
5466	/**
5467	* Reads a guest segment register from the current VMCS into the guest-CPU
5468	* context.
5469	*
5470	* @returns VBox status code.
5471	* @param pVCpu Pointer to the VMCPU.
5472	* @param idxSel Index of the selector in the VMCS.
5473	* @param idxLimit Index of the segment limit in the VMCS.
5474	* @param idxBase Index of the segment base in the VMCS.
5475	* @param idxAccess Index of the access rights of the segment in the VMCS.
5476	* @param pSelReg Pointer to the segment selector.
5477	*
5478	* @remarks No-long-jump zone!!!
5479	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5480	* macro as that takes care of whether to read from the VMCS cache or
5481	* not.
5482	*/
5483	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5484	PCPUMSELREG pSelReg)
5485	{
5486	uint32_t u32Val = 0;
5487	int rc = VMXReadVmcs32(idxSel, &u32Val);
5488	AssertRCReturn(rc, rc);
5489	pSelReg->Sel = (uint16_t)u32Val;
5490	pSelReg->ValidSel = (uint16_t)u32Val;
5491	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5492
5493	rc = VMXReadVmcs32(idxLimit, &u32Val);
5494	AssertRCReturn(rc, rc);
5495	pSelReg->u32Limit = u32Val;
5496
5497	uint64_t u64Val = 0;
5498	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5499	AssertRCReturn(rc, rc);
5500	pSelReg->u64Base = u64Val;
5501
5502	rc = VMXReadVmcs32(idxAccess, &u32Val);
5503	AssertRCReturn(rc, rc);
5504	pSelReg->Attr.u = u32Val;
5505
5506	/*
5507	* If VT-x marks the segment as unusable, most other bits remain undefined:
5508	* - For CS the L, D and G bits have meaning.
5509	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5510	* - For the remaining data segments no bits are defined.
5511	*
5512	* The present bit and the unusable bit has been observed to be set at the
5513	* same time (the selector was supposed to invalid as we started executing
5514	* a V8086 interrupt in ring-0).
5515	*
5516	* What should be important for the rest of the VBox code that the P bit is
5517	* cleared. Some of the other VBox code recognizes the unusable bit, but
5518	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5519	* safe side here, we'll strip off P and other bits we don't care about. If
5520	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5521	*
5522	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5523	*/
5524	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5525	{
5526	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5527
5528	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5529	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5530	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5531
5532	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5533	#ifdef DEBUG_bird
5534	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5535	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5536	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5537	#endif
5538	}
5539	return VINF_SUCCESS;
5540	}
5541
5542
5543	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5544	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5545	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5546	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5547	#else
5548	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5549	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5550	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5551	#endif
5552
5553
5554	/**
5555	* Saves the guest segment registers from the current VMCS into the guest-CPU
5556	* context.
5557	*
5558	* @returns VBox status code.
5559	* @param pVCpu Pointer to the VMCPU.
5560	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5561	* out-of-sync. Make sure to update the required fields
5562	* before using them.
5563	*
5564	* @remarks No-long-jump zone!!!
5565	*/
5566	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5567	{
5568	/* Guest segment registers. */
5569	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5570	{
5571	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5572	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5573	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5574	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5575	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5576	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5577	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5578
5579	/* Restore segment attributes for real-on-v86 mode hack. */
5580	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5581	{
5582	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5583	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5584	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5585	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5586	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5587	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5588	}
5589	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5590	}
5591
5592	return VINF_SUCCESS;
5593	}
5594
5595
5596	/**
5597	* Saves the guest descriptor table registers and task register from the current
5598	* VMCS into the guest-CPU context.
5599	*
5600	* @returns VBox status code.
5601	* @param pVCpu Pointer to the VMCPU.
5602	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5603	* out-of-sync. Make sure to update the required fields
5604	* before using them.
5605	*
5606	* @remarks No-long-jump zone!!!
5607	*/
5608	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5609	{
5610	int rc = VINF_SUCCESS;
5611
5612	/* Guest LDTR. */
5613	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5614	{
5615	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5616	AssertRCReturn(rc, rc);
5617	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5618	}
5619
5620	/* Guest GDTR. */
5621	uint64_t u64Val = 0;
5622	uint32_t u32Val = 0;
5623	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5624	{
5625	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5626	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5627	pMixedCtx->gdtr.pGdt = u64Val;
5628	pMixedCtx->gdtr.cbGdt = u32Val;
5629	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5630	}
5631
5632	/* Guest IDTR. */
5633	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5634	{
5635	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5636	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5637	pMixedCtx->idtr.pIdt = u64Val;
5638	pMixedCtx->idtr.cbIdt = u32Val;
5639	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5640	}
5641
5642	/* Guest TR. */
5643	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5644	{
5645	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5646	AssertRCReturn(rc, rc);
5647
5648	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5649	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5650	{
5651	rc = VMXLOCAL_READ_SEG(TR, tr);
5652	AssertRCReturn(rc, rc);
5653	}
5654	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5655	}
5656	return rc;
5657	}
5658
5659	#undef VMXLOCAL_READ_SEG
5660
5661
5662	/**
5663	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5664	* context.
5665	*
5666	* @returns VBox status code.
5667	* @param pVCpu Pointer to the VMCPU.
5668	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5669	* out-of-sync. Make sure to update the required fields
5670	* before using them.
5671	*
5672	* @remarks No-long-jump zone!!!
5673	*/
5674	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5675	{
5676	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5677	{
5678	if (!CPUMIsHyperDebugStateActive(pVCpu))
5679	{
5680	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5681	uint32_t u32Val;
5682	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5683	pMixedCtx->dr[7] = u32Val;
5684	}
5685
5686	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5687	}
5688	return VINF_SUCCESS;
5689	}
5690
5691
5692	/**
5693	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5694	*
5695	* @returns VBox status code.
5696	* @param pVCpu Pointer to the VMCPU.
5697	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5698	* out-of-sync. Make sure to update the required fields
5699	* before using them.
5700	*
5701	* @remarks No-long-jump zone!!!
5702	*/
5703	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5704	{
5705	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5706	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5707	return VINF_SUCCESS;
5708	}
5709
5710
5711	/**
5712	* Saves the entire guest state from the currently active VMCS into the
5713	* guest-CPU context. This essentially VMREADs all guest-data.
5714	*
5715	* @returns VBox status code.
5716	* @param pVCpu Pointer to the VMCPU.
5717	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5718	* out-of-sync. Make sure to update the required fields
5719	* before using them.
5720	*/
5721	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5722	{
5723	Assert(pVCpu);
5724	Assert(pMixedCtx);
5725
5726	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5727	return VINF_SUCCESS;
5728
5729	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5730	there is no real need to. */
5731	if (VMMRZCallRing3IsEnabled(pVCpu))
5732	VMMR0LogFlushDisable(pVCpu);
5733	else
5734	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5735	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5736
5737	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5738	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5739
5740	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5741	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5742
5743	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5744	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5745
5746	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5747	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5748
5749	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5750	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5751
5752	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5753	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5754
5755	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5756	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5757
5758	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5759	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5760
5761	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5762	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5763
5764	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5765	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5766
5767	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5768	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5769
5770	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5771	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5772
5773	if (VMMRZCallRing3IsEnabled(pVCpu))
5774	VMMR0LogFlushEnable(pVCpu);
5775
5776	return rc;
5777	}
5778
5779
5780	/**
5781	* Check per-VM and per-VCPU force flag actions that require us to go back to
5782	* ring-3 for one reason or another.
5783	*
5784	* @returns VBox status code (information status code included).
5785	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5786	* ring-3.
5787	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5788	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5789	* interrupts)
5790	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5791	* all EMTs to be in ring-3.
5792	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5793	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5794	* to the EM loop.
5795	*
5796	* @param pVM Pointer to the VM.
5797	* @param pVCpu Pointer to the VMCPU.
5798	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5799	* out-of-sync. Make sure to update the required fields
5800	* before using them.
5801	*/
5802	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5803	{
5804	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5805
5806	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5807	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5808	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5809	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5810	{
5811	/* We need the control registers now, make sure the guest-CPU context is updated. */
5812	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5813	AssertRCReturn(rc3, rc3);
5814
5815	/* Pending HM CR3 sync. */
5816	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5817	{
5818	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5819	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
5820	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5821	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5822	}
5823
5824	/* Pending HM PAE PDPEs. */
5825	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5826	{
5827	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5828	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5829	}
5830
5831	/* Pending PGM C3 sync. */
5832	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5833	{
5834	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5835	if (rc2 != VINF_SUCCESS)
5836	{
5837	AssertRC(rc2);
5838	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5839	return rc2;
5840	}
5841	}
5842
5843	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5844	/* -XXX- what was that about single stepping? */
5845	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5846	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5847	{
5848	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5849	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5850	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5851	return rc2;
5852	}
5853
5854	/* Pending VM request packets, such as hardware interrupts. */
5855	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5856	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5857	{
5858	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5859	return VINF_EM_PENDING_REQUEST;
5860	}
5861
5862	/* Pending PGM pool flushes. */
5863	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5864	{
5865	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5866	return VINF_PGM_POOL_FLUSH_PENDING;
5867	}
5868
5869	/* Pending DMA requests. */
5870	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5871	{
5872	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5873	return VINF_EM_RAW_TO_R3;
5874	}
5875	}
5876
5877	/* Paranoia. */
5878	return VINF_SUCCESS;
5879	}
5880
5881
5882	/**
5883	* Converts any TRPM trap into a pending HM event. This is typically used when
5884	* entering from ring-3 (not longjmp returns).
5885	*
5886	* @param pVCpu Pointer to the VMCPU.
5887	*/
5888	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5889	{
5890	Assert(TRPMHasTrap(pVCpu));
5891	Assert(!pVCpu->hm.s.Event.fPending);
5892
5893	uint8_t uVector;
5894	TRPMEVENT enmTrpmEvent;
5895	RTGCUINT uErrCode;
5896	RTGCUINTPTR GCPtrFaultAddress;
5897	uint8_t cbInstr;
5898
5899	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5900	AssertRC(rc);
5901
5902	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5903	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5904	if (enmTrpmEvent == TRPM_TRAP)
5905	{
5906	switch (uVector)
5907	{
5908	case X86_XCPT_BP:
5909	case X86_XCPT_OF:
5910	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5911	break;
5912
5913	case X86_XCPT_PF:
5914	case X86_XCPT_DF:
5915	case X86_XCPT_TS:
5916	case X86_XCPT_NP:
5917	case X86_XCPT_SS:
5918	case X86_XCPT_GP:
5919	case X86_XCPT_AC:
5920	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5921	/* no break! */
5922	default:
5923	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5924	break;
5925	}
5926	}
5927	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5928	{
5929	if (uVector == X86_XCPT_NMI)
5930	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5931	else
5932	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5933	}
5934	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5935	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5936	else
5937	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5938
5939	rc = TRPMResetTrap(pVCpu);
5940	AssertRC(rc);
5941	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5942	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5943
5944	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5945	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
5946	}
5947
5948
5949	/**
5950	* Converts any pending HM event into a TRPM trap. Typically used when leaving
5951	* VT-x to execute any instruction.
5952	*
5953	* @param pvCpu Pointer to the VMCPU.
5954	*/
5955	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5956	{
5957	Assert(pVCpu->hm.s.Event.fPending);
5958
5959	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5960	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5961	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5962	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5963
5964	/* If a trap was already pending, we did something wrong! */
5965	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5966
5967	TRPMEVENT enmTrapType;
5968	switch (uVectorType)
5969	{
5970	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5971	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5972	enmTrapType = TRPM_HARDWARE_INT;
5973	break;
5974
5975	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5976	enmTrapType = TRPM_SOFTWARE_INT;
5977	break;
5978
5979	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5980	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5981	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5982	enmTrapType = TRPM_TRAP;
5983	break;
5984
5985	default:
5986	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5987	enmTrapType = TRPM_32BIT_HACK;
5988	break;
5989	}
5990
5991	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5992
5993	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5994	AssertRC(rc);
5995
5996	if (fErrorCodeValid)
5997	TRPMSetErrorCode(pVCpu, uErrorCode);
5998
5999	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6000	&& uVector == X86_XCPT_PF)
6001	{
6002	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6003	}
6004	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6005	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6006	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6007	{
6008	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6009	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6010	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6011	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6012	}
6013	pVCpu->hm.s.Event.fPending = false;
6014	}
6015
6016
6017	/**
6018	* Does the necessary state syncing before returning to ring-3 for any reason
6019	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6020	*
6021	* @param pVM Pointer to the VM.
6022	* @param pVCpu Pointer to the VMCPU.
6023	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6024	* out-of-sync. Make sure to update the required fields
6025	* before using them.
6026	*
6027	* @remarks No-long-jmp zone!!!
6028	*/
6029	static void hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6030	{
6031	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6032	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6033
6034	/* Avoid repeating this work when thread-context hooks are used and we had been preempted before
6035	which would've done this work from the VMXR0ThreadCtxCallback(). */
6036	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
6037	bool fPreemptDisabled = false;
6038	if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
6039	{
6040	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6041	RTThreadPreemptDisable(&PreemptState);
6042	fPreemptDisabled = true;
6043	if (pVCpu->hm.s.fLeaveDone)
6044	{
6045	RTThreadPreemptRestore(&PreemptState);
6046	return;
6047	}
6048	}
6049
6050	/* Save the guest state if necessary. */
6051	if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6052	{
6053	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6054	AssertRC(rc);
6055	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6056	}
6057
6058	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6059	if (CPUMIsGuestFPUStateActive(pVCpu))
6060	{
6061	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6062	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6063	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
6064	}
6065
6066	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6067	#ifdef VBOX_STRICT
6068	if (CPUMIsHyperDebugStateActive(pVCpu))
6069	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6070	#endif
6071	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6072	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
6073	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6074	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6075
6076	/* Restore host-state bits that VT-x only restores partially. */
6077	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6078	{
6079	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6080	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6081	}
6082
6083	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6084	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6085	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6086	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6087	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6088	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6089	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6090	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6091
6092	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6093
6094	/* Restore preemption if we previous disabled it ourselves. */
6095	if (fPreemptDisabled)
6096	{
6097	pVCpu->hm.s.fLeaveDone = true;
6098	RTThreadPreemptRestore(&PreemptState);
6099	}
6100	}
6101
6102
6103	/**
6104	* Does the necessary state syncing before doing a longjmp to ring-3.
6105	*
6106	* @param pVM Pointer to the VM.
6107	* @param pVCpu Pointer to the VMCPU.
6108	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6109	* out-of-sync. Make sure to update the required fields
6110	* before using them.
6111	*
6112	* @remarks No-long-jmp zone!!!
6113	*/
6114	DECLINLINE(void) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6115	{
6116	hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6117	}
6118
6119
6120	/**
6121	* Take necessary actions before going back to ring-3.
6122	*
6123	* An action requires us to go back to ring-3. This function does the necessary
6124	* steps before we can safely return to ring-3. This is not the same as longjmps
6125	* to ring-3, this is voluntary and prepares the guest so it may continue
6126	* executing outside HM (recompiler/IEM).
6127	*
6128	* @param pVM Pointer to the VM.
6129	* @param pVCpu Pointer to the VMCPU.
6130	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6131	* out-of-sync. Make sure to update the required fields
6132	* before using them.
6133	* @param rcExit The reason for exiting to ring-3. Can be
6134	* VINF_VMM_UNKNOWN_RING3_CALL.
6135	*/
6136	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6137	{
6138	Assert(pVM);
6139	Assert(pVCpu);
6140	Assert(pMixedCtx);
6141	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6142
6143	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6144	{
6145	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
6146	return;
6147	}
6148	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6149	{
6150	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6151	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6152	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6153	pVCpu->hm.s.vmx.LastError.idCurrentCpu = RTMpCpuId();
6154	return;
6155	}
6156
6157	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6158	VMMRZCallRing3Disable(pVCpu);
6159	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6160
6161	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6162	if (pVCpu->hm.s.Event.fPending)
6163	{
6164	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6165	Assert(!pVCpu->hm.s.Event.fPending);
6166	}
6167
6168	/* Save guest state and restore host state bits. */
6169	hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6170	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6171
6172	/* Sync recompiler state. */
6173	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6174	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6175	\| CPUM_CHANGED_LDTR
6176	\| CPUM_CHANGED_GDTR
6177	\| CPUM_CHANGED_IDTR
6178	\| CPUM_CHANGED_TR
6179	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6180	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6181	if ( pVM->hm.s.fNestedPaging
6182	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6183	{
6184	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6185	}
6186
6187	/*
6188	* Clear the X86_EFL_TF if necessary .
6189	*/
6190	if (pVCpu->hm.s.fClearTrapFlag)
6191	{
6192	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6193	pMixedCtx->eflags.Bits.u1TF = 0;
6194	pVCpu->hm.s.fClearTrapFlag = false;
6195	}
6196	/** @todo there seems to be issues with the resume flag when the monitor trap
6197	* flag is pending without being used. Seen early in bios init when
6198	* accessing APIC page in prot mode. */
6199
6200	/* On our way back from ring-3 the following needs to be done. */
6201	/** @todo This can change with preemption hooks. */
6202	if (rcExit == VINF_EM_RAW_INTERRUPT)
6203	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
6204	else
6205	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
6206
6207	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6208	VMMRZCallRing3Enable(pVCpu);
6209	}
6210
6211
6212	/**
6213	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6214	* longjump to ring-3 and possibly get preempted.
6215	*
6216	* @param pVCpu Pointer to the VMCPU.
6217	* @param enmOperation The operation causing the ring-3 longjump.
6218	* @param pvUser The user argument (pointer to the possibly
6219	* out-of-date guest-CPU context).
6220	*
6221	* @remarks Must never be called with @a enmOperation ==
6222	* VMMCALLRING3_VM_R0_ASSERTION.
6223	*/
6224	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6225	{
6226	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6227	Assert(pVCpu);
6228	Assert(pvUser);
6229	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6230	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6231
6232	VMMRZCallRing3Disable(pVCpu);
6233	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6234
6235	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6236	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6237
6238	VMMRZCallRing3Enable(pVCpu);
6239	}
6240
6241
6242	/**
6243	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6244	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6245	*
6246	* @param pVCpu Pointer to the VMCPU.
6247	*/
6248	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6249	{
6250	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6251	{
6252	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6253	{
6254	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6255	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6256	AssertRC(rc);
6257	}
6258	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6259	}
6260
6261
6262	/**
6263	* Injects any pending events into the guest if the guest is in a state to
6264	* receive them.
6265	*
6266	* @returns VBox status code (informational status codes included).
6267	* @param pVCpu Pointer to the VMCPU.
6268	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6269	* out-of-sync. Make sure to update the required fields
6270	* before using them.
6271	*/
6272	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6273	{
6274	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6275	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6276	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6277	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6278
6279	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6280	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6281	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6282	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6283	Assert(!TRPMHasTrap(pVCpu));
6284
6285	int rc = VINF_SUCCESS;
6286	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6287	{
6288	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6289	bool fInject = true;
6290	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6291	{
6292	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6293	AssertRCReturn(rc, rc);
6294	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6295	if ( fBlockInt
6296	\|\| fBlockSti
6297	\|\| fBlockMovSS)
6298	{
6299	fInject = false;
6300	}
6301	}
6302	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6303	&& ( fBlockMovSS
6304	\|\| fBlockSti))
6305	{
6306	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6307	fInject = false;
6308	}
6309
6310	if (fInject)
6311	{
6312	Log4(("Injecting pending event vcpu[%RU32]\n", pVCpu->idCpu));
6313	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6314	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6315	AssertRCReturn(rc, rc);
6316	pVCpu->hm.s.Event.fPending = false;
6317
6318	#ifdef VBOX_WITH_STATISTICS
6319	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6320	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6321	else
6322	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6323	#endif
6324	}
6325	else
6326	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6327	} /** @todo SMI. SMIs take priority over NMIs. */
6328	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6329	{
6330	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6331	if ( !fBlockMovSS
6332	&& !fBlockSti)
6333	{
6334	Log4(("Injecting NMI\n"));
6335	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6336	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6337	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6338	0 /* GCPtrFaultAddress */, &uIntrState);
6339	AssertRCReturn(rc, rc);
6340	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6341
6342	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6343	}
6344	else
6345	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6346	}
6347	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6348	&& !pVCpu->hm.s.fSingleInstruction)
6349	{
6350	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6351	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6352	AssertRCReturn(rc, rc);
6353	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6354	if ( !fBlockInt
6355	&& !fBlockSti
6356	&& !fBlockMovSS)
6357	{
6358	uint8_t u8Interrupt;
6359	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6360	if (RT_SUCCESS(rc))
6361	{
6362	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6363	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6364	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6365	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6366	0 /* GCPtrFaultAddress */, &uIntrState);
6367
6368	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6369	}
6370	else
6371	{
6372	/** @todo Does this actually happen? If not turn it into an assertion. */
6373	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6374	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6375	rc = VINF_SUCCESS;
6376	}
6377	}
6378	else
6379	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6380	}
6381
6382	/*
6383	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6384	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6385	*/
6386	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6387	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6388	int rc2 = VINF_SUCCESS;
6389	if ( fBlockSti
6390	\|\| fBlockMovSS)
6391	{
6392	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6393	{
6394	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6395	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6396	{
6397	/*
6398	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6399	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6400	* See Intel spec. 27.3.4 "Saving Non-Register State".
6401	*/
6402	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6403	AssertRCReturn(rc, rc);
6404	}
6405	}
6406	else
6407	{
6408	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6409	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6410	uIntrState = 0;
6411	}
6412	}
6413
6414	/*
6415	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6416	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6417	*/
6418	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6419	AssertRC(rc2);
6420
6421	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6422	return rc;
6423	}
6424
6425
6426	/**
6427	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6428	*
6429	* @param pVCpu Pointer to the VMCPU.
6430	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6431	* out-of-sync. Make sure to update the required fields
6432	* before using them.
6433	*/
6434	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6435	{
6436	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6437	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6438	}
6439
6440
6441	/**
6442	* Injects a double-fault (#DF) exception into the VM.
6443	*
6444	* @returns VBox status code (informational status code included).
6445	* @param pVCpu Pointer to the VMCPU.
6446	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6447	* out-of-sync. Make sure to update the required fields
6448	* before using them.
6449	*/
6450	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6451	{
6452	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6453	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6454	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6455	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6456	puIntrState);
6457	}
6458
6459
6460	/**
6461	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6462	*
6463	* @param pVCpu Pointer to the VMCPU.
6464	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6465	* out-of-sync. Make sure to update the required fields
6466	* before using them.
6467	*/
6468	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6469	{
6470	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6471	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6472	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6473	}
6474
6475
6476	/**
6477	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6478	*
6479	* @param pVCpu Pointer to the VMCPU.
6480	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6481	* out-of-sync. Make sure to update the required fields
6482	* before using them.
6483	* @param cbInstr The value of RIP that is to be pushed on the guest
6484	* stack.
6485	*/
6486	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6487	{
6488	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6489	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6490	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6491	}
6492
6493
6494	/**
6495	* Injects a general-protection (#GP) fault into the VM.
6496	*
6497	* @returns VBox status code (informational status code included).
6498	* @param pVCpu Pointer to the VMCPU.
6499	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6500	* out-of-sync. Make sure to update the required fields
6501	* before using them.
6502	* @param u32ErrorCode The error code associated with the #GP.
6503	*/
6504	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6505	uint32_t *puIntrState)
6506	{
6507	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6508	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6509	if (fErrorCodeValid)
6510	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6511	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6512	puIntrState);
6513	}
6514
6515
6516	/**
6517	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6518	*
6519	* @param pVCpu Pointer to the VMCPU.
6520	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6521	* out-of-sync. Make sure to update the required fields
6522	* before using them.
6523	* @param uVector The software interrupt vector number.
6524	* @param cbInstr The value of RIP that is to be pushed on the guest
6525	* stack.
6526	*/
6527	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6528	{
6529	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6530	if ( uVector == X86_XCPT_BP
6531	\|\| uVector == X86_XCPT_OF)
6532	{
6533	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6534	}
6535	else
6536	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6537	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6538	}
6539
6540
6541	/**
6542	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6543	* stack.
6544	*
6545	* @returns VBox status code (information status code included).
6546	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6547	* @param pVM Pointer to the VM.
6548	* @param pMixedCtx Pointer to the guest-CPU context.
6549	* @param uValue The value to push to the guest stack.
6550	*/
6551	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6552	{
6553	/*
6554	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6555	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6556	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6557	*/
6558	if (pMixedCtx->sp == 1)
6559	return VINF_EM_RESET;
6560	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6561	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6562	AssertRCReturn(rc, rc);
6563	return rc;
6564	}
6565
6566
6567	/**
6568	* Injects an event into the guest upon VM-entry by updating the relevant fields
6569	* in the VM-entry area in the VMCS.
6570	*
6571	* @returns VBox status code (informational error codes included).
6572	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6573	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6574	*
6575	* @param pVCpu Pointer to the VMCPU.
6576	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6577	* be out-of-sync. Make sure to update the required
6578	* fields before using them.
6579	* @param u64IntrInfo The VM-entry interruption-information field.
6580	* @param cbInstr The VM-entry instruction length in bytes (for
6581	* software interrupts, exceptions and privileged
6582	* software exceptions).
6583	* @param u32ErrCode The VM-entry exception error code.
6584	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6585	* @param puIntrState Pointer to the current guest interruptibility-state.
6586	* This interruptibility-state will be updated if
6587	* necessary. This cannot not be NULL.
6588	*
6589	* @remarks No-long-jump zone!!!
6590	* @remarks Requires CR0!
6591	*/
6592	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6593	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6594	{
6595	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6596	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6597	Assert(puIntrState);
6598	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6599
6600	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6601	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6602
6603	#ifdef VBOX_STRICT
6604	/* Validate the error-code-valid bit for hardware exceptions. */
6605	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6606	{
6607	switch (uVector)
6608	{
6609	case X86_XCPT_PF:
6610	case X86_XCPT_DF:
6611	case X86_XCPT_TS:
6612	case X86_XCPT_NP:
6613	case X86_XCPT_SS:
6614	case X86_XCPT_GP:
6615	case X86_XCPT_AC:
6616	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6617	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6618	/* fallthru */
6619	default:
6620	break;
6621	}
6622	}
6623	#endif
6624
6625	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6626	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6627	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6628
6629	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6630
6631	/* We require CR0 to check if the guest is in real-mode. */
6632	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6633	AssertRCReturn(rc, rc);
6634
6635	/*
6636	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6637	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6638	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6639	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6640	*/
6641	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6642	{
6643	PVM pVM = pVCpu->CTX_SUFF(pVM);
6644	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6645	{
6646	Assert(PDMVmmDevHeapIsEnabled(pVM));
6647	Assert(pVM->hm.s.vmx.pRealModeTSS);
6648
6649	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6650	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6651	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6652	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6653	AssertRCReturn(rc, rc);
6654	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6655
6656	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6657	const size_t cbIdtEntry = 4;
6658	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6659	{
6660	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6661	if (uVector == X86_XCPT_DF)
6662	return VINF_EM_RESET;
6663	else if (uVector == X86_XCPT_GP)
6664	{
6665	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6666	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6667	}
6668
6669	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6670	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6671	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6672	}
6673
6674	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6675	uint16_t uGuestIp = pMixedCtx->ip;
6676	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6677	{
6678	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6679	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6680	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6681	}
6682	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6683	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6684
6685	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6686	uint16_t offIdtEntry = 0;
6687	RTSEL selIdtEntry = 0;
6688	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6689	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6690	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6691	AssertRCReturn(rc, rc);
6692
6693	/* Construct the stack frame for the interrupt/exception handler. */
6694	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6695	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6696	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6697	AssertRCReturn(rc, rc);
6698
6699	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6700	if (rc == VINF_SUCCESS)
6701	{
6702	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6703	pMixedCtx->rip = offIdtEntry;
6704	pMixedCtx->cs.Sel = selIdtEntry;
6705	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6706	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6707	&& uVector == X86_XCPT_PF)
6708	{
6709	pMixedCtx->cr2 = GCPtrFaultAddress;
6710	}
6711	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6712	\| HM_CHANGED_GUEST_RIP
6713	\| HM_CHANGED_GUEST_RFLAGS
6714	\| HM_CHANGED_GUEST_RSP;
6715
6716	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6717	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6718	{
6719	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6720	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6721	Log4(("Clearing inhibition due to STI.\n"));
6722	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6723	}
6724	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6725	}
6726	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6727	return rc;
6728	}
6729	else
6730	{
6731	/*
6732	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6733	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6734	*/
6735	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6736	}
6737	}
6738
6739	/* Validate. */
6740	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6741	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6742	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6743
6744	/* Inject. */
6745	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6746	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6747	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6748	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6749
6750	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6751	&& uVector == X86_XCPT_PF)
6752	{
6753	pMixedCtx->cr2 = GCPtrFaultAddress;
6754	}
6755
6756	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6757	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6758
6759	AssertRCReturn(rc, rc);
6760	return rc;
6761	}
6762
6763
6764	/**
6765	* Enters the VT-x session.
6766	*
6767	* @returns VBox status code.
6768	* @param pVM Pointer to the VM.
6769	* @param pVCpu Pointer to the VMCPU.
6770	* @param pCpu Pointer to the CPU info struct.
6771	*/
6772	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6773	{
6774	AssertPtr(pVM);
6775	AssertPtr(pVCpu);
6776	Assert(pVM->hm.s.vmx.fSupported);
6777	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6778	NOREF(pCpu);
6779
6780	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6781
6782	#ifdef VBOX_STRICT
6783	/* Make sure we're in VMX root mode. */
6784	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6785	if (!(u32HostCR4 & X86_CR4_VMXE))
6786	{
6787	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6788	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6789	}
6790	#endif
6791
6792	/* Load the active VMCS as the current one. */
6793	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6794	if (RT_FAILURE(rc))
6795	return rc;
6796
6797	/** @todo this will change with preemption hooks where can VMRESUME as long
6798	* as we're no preempted. */
6799	pVCpu->hm.s.fResumeVM = false;
6800	pVCpu->hm.s.fLeaveDone = false;
6801	return VINF_SUCCESS;
6802	}
6803
6804
6805	/**
6806	* The thread-context callback (only on platforms which support it).
6807	*
6808	* @param enmEvent The thread-context event.
6809	* @param pVCpu Pointer to the VMCPU.
6810	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
6811	* @thread EMT.
6812	*/
6813	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
6814	{
6815	switch (enmEvent)
6816	{
6817	case RTTHREADCTXEVENT_PREEMPTING:
6818	{
6819	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6820	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu)); /* Paranoia. */
6821
6822	PVM pVM = pVCpu->CTX_SUFF(pVM);
6823	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
6824	VMMRZCallRing3Disable(pVCpu); /* No longjmps (log-flush, locks) in this fragile context. */
6825	hmR0VmxLeave(pVM, pVCpu, pMixedCtx); /* Save the guest-state, restore host-state (FPU, debug etc.). */
6826
6827	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs); /* Flush VMCS CPU state to VMCS region in memory. */
6828	AssertRC(rc); NOREF(rc);
6829
6830	rc = HMR0LeaveEx(pVCpu); /* Leave HM context, takes care of local init (term). */
6831	AssertRC(rc); NOREF(rc);
6832
6833	VMMRZCallRing3Enable(pVCpu); /* Restore longjmp state. */
6834	break;
6835	}
6836
6837	case RTTHREADCTXEVENT_RESUMED:
6838	{
6839	/* Disable preemption, we don't want to be migrated to another CPU while re-initializing VT-x state. */
6840	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
6841	RTThreadPreemptDisable(&PreemptState);
6842
6843	/* Initialize the bare minimum state required for HM. This takes care of
6844	initializing VT-x if necessary (onlined CPUs, local init etc.) */
6845	HMR0EnterEx(pVCpu);
6846	Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_GUEST_CR0));
6847
6848	/* Load the active VMCS as the current one. */
6849	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6850	AssertRC(rc);
6851
6852	pVCpu->hm.s.fResumeVM = false;
6853	pVCpu->hm.s.fLeaveDone = false;
6854
6855	/* Restore preemption, migrating to another CPU should be fine now. */
6856	RTThreadPreemptRestore(&PreemptState);
6857	break;
6858	}
6859
6860	default:
6861	break;
6862	}
6863	}
6864
6865
6866	/**
6867	* Leaves the VT-x session.
6868	*
6869	* @returns VBox status code.
6870	* @param pVM Pointer to the VM.
6871	* @param pVCpu Pointer to the VMCPU.
6872	* @param pCtx Pointer to the guest-CPU context.
6873	*/
6874	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6875	{
6876	AssertPtr(pVCpu);
6877	NOREF(pVM);
6878	NOREF(pCtx);
6879
6880	if (!VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6881	{
6882	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6883
6884	/*
6885	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6886	* and mark the VMCS launch-state as "clear".
6887	*/
6888	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6889	return rc;
6890	}
6891
6892	/* With thread-context hooks, nothing to do here. It's taken care of in VMXR0ThreadCtxCallback(). */
6893	return VINF_SUCCESS;
6894	}
6895
6896
6897	/**
6898	* Saves the host state in the VMCS host-state.
6899	* Sets up the VM-exit MSR-load area.
6900	*
6901	* The CPU state will be loaded from these fields on every successful VM-exit.
6902	*
6903	* @returns VBox status code.
6904	* @param pVM Pointer to the VM.
6905	* @param pVCpu Pointer to the VMCPU.
6906	*
6907	* @remarks No-long-jump zone!!!
6908	*/
6909	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6910	{
6911	AssertPtr(pVM);
6912	AssertPtr(pVCpu);
6913	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6914
6915	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6916
6917	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6918	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6919	return VINF_SUCCESS;
6920
6921	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6922	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6923
6924	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6925	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6926
6927	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6928	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6929
6930	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6931	return rc;
6932	}
6933
6934
6935	/**
6936	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6937	* loaded from these fields on every successful VM-entry.
6938	*
6939	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6940	* Sets up the VM-entry controls.
6941	* Sets up the appropriate VMX non-root function to execute guest code based on
6942	* the guest CPU mode.
6943	*
6944	* @returns VBox status code.
6945	* @param pVM Pointer to the VM.
6946	* @param pVCpu Pointer to the VMCPU.
6947	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6948	* out-of-sync. Make sure to update the required fields
6949	* before using them.
6950	*
6951	* @remarks No-long-jump zone!!!
6952	*/
6953	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6954	{
6955	AssertPtr(pVM);
6956	AssertPtr(pVCpu);
6957	AssertPtr(pMixedCtx);
6958	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6959
6960	#ifdef LOG_ENABLED
6961	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
6962	* probably not initialized yet? Anyway this will do for now. */
6963	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
6964	VMMR0LogFlushDisable(pVCpu);
6965	#endif
6966
6967	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6968
6969	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6970
6971	/* Determine real-on-v86 mode. */
6972	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6973	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6974	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6975	{
6976	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6977	}
6978
6979	/*
6980	* Load the guest-state into the VMCS.
6981	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6982	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6983	*/
6984	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6985	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6986
6987	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6988	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6989
6990	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6991	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6992
6993	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6994	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6995
6996	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6997	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6998	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6999
7000	rc = hmR0VmxLoadGuestDebugState(pVCpu, pMixedCtx);
7001	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugState: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7002
7003	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7004	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7005
7006	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7007	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7008
7009	/* Must be done after hmR0VmxLoadGuestDebugState() as it may have updated eflags.TF for debugging purposes. */
7010	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7011	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7012
7013	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7014	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7015
7016	/* Clear any unused and reserved bits. */
7017	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7018
7019	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
7020	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
7021	pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
7022
7023	#ifdef LOG_ENABLED
7024	/* Only reenable log-flushing if the caller has it enabled. */
7025	if (!fCallerDisabledLogFlush)
7026	VMMR0LogFlushEnable(pVCpu);
7027	#endif
7028
7029	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7030	return rc;
7031	}
7032
7033
7034	/**
7035	* Loads the guest state into the VMCS guest-state area.
7036	*
7037	* @returns VBox status code.
7038	* @param pVM Pointer to the VM.
7039	* @param pVCpu Pointer to the VMCPU.
7040	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7041	* out-of-sync. Make sure to update the required fields
7042	* before using them.
7043	*
7044	* @remarks No-long-jump zone!!!
7045	*/
7046	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7047	{
7048	/*
7049	* Avoid reloading the guest state on longjmp reentrants and do it lazily just before executing the guest.
7050	* This only helps when we get rescheduled more than once to a different host CPU on a longjmp trip before
7051	* finally executing guest code.
7052	*/
7053	return VINF_SUCCESS;
7054	}
7055
7056
7057	/**
7058	* Does the preparations before executing guest code in VT-x.
7059	*
7060	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7061	* recompiler. We must be cautious what we do here regarding committing
7062	* guest-state information into the VMCS assuming we assuredly execute the
7063	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
7064	* clearing the common-state (TRPM/forceflags), we must undo those changes so
7065	* that the recompiler can (and should) use them when it resumes guest
7066	* execution. Otherwise such operations must be done when we can no longer
7067	* exit to ring-3.
7068	*
7069	* @returns VBox status code (informational status codes included).
7070	* @retval VINF_SUCCESS if we can proceed with running the guest.
7071	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
7072	* into the guest.
7073	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7074	*
7075	* @param pVM Pointer to the VM.
7076	* @param pVCpu Pointer to the VMCPU.
7077	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7078	* out-of-sync. Make sure to update the required fields
7079	* before using them.
7080	* @param pVmxTransient Pointer to the VMX transient structure.
7081	*
7082	* @remarks Called with preemption disabled.
7083	*/
7084	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7085	{
7086	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7087
7088	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7089	PGMRZDynMapFlushAutoSet(pVCpu);
7090	#endif
7091
7092	/* Check force flag actions that might require us to go back to ring-3. */
7093	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7094	if (rc != VINF_SUCCESS)
7095	return rc;
7096
7097	#ifndef IEM_VERIFICATION_MODE_FULL
7098	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7099	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7100	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7101	{
7102	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7103	RTGCPHYS GCPhysApicBase;
7104	GCPhysApicBase = pMixedCtx->msrApicBase;
7105	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7106
7107	/* Unalias any existing mapping. */
7108	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7109	AssertRCReturn(rc, rc);
7110
7111	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7112	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7113	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7114	AssertRCReturn(rc, rc);
7115
7116	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7117	}
7118	#endif /* !IEM_VERIFICATION_MODE_FULL */
7119
7120	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7121	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
7122	pVmxTransient->uEflags = ASMIntDisableFlags();
7123	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7124	{
7125	ASMSetFlags(pVmxTransient->uEflags);
7126	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7127	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
7128	return VINF_EM_RAW_INTERRUPT;
7129	}
7130	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7131	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7132	#endif
7133
7134	/*
7135	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
7136	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
7137	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
7138	*/
7139	/** @todo Rework event evaluation and injection to be completely separate.
7140	* Update: Tried it, problem with handling halts. Control never returns to VT-x
7141	* if we exit VT-x with external interrupt pending in a TRPM event. */
7142	if (TRPMHasTrap(pVCpu))
7143	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7144
7145	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7146	AssertRCReturn(rc, rc);
7147	return rc;
7148	}
7149
7150
7151	/**
7152	* Prepares to run guest code in VT-x and we've committed to doing so. This
7153	* means there is no backing out to ring-3 or anywhere else at this
7154	* point.
7155	*
7156	* @param pVM Pointer to the VM.
7157	* @param pVCpu Pointer to the VMCPU.
7158	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7159	* out-of-sync. Make sure to update the required fields
7160	* before using them.
7161	* @param pVmxTransient Pointer to the VMX transient structure.
7162	*
7163	* @remarks Called with preemption disabled.
7164	* @remarks No-long-jump zone!!!
7165	*/
7166	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7167	{
7168	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7169	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7170
7171	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7172	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
7173	pVmxTransient->uEflags = ASMIntDisableFlags();
7174	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7175	#endif
7176
7177	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
7178	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7179	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7180	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7181	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
7182	#endif
7183	int rc = VINF_SUCCESS;
7184	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7185	{
7186	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7187	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7188	}
7189	else if (pVCpu->hm.s.fContextUseFlags)
7190	{
7191	rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7192	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7193	}
7194	AssertRC(rc);
7195	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
7196
7197	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7198	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7199	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7200	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7201	#endif
7202
7203	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
7204	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7205	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7206
7207	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7208	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
7209	{
7210	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7211	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7212	}
7213
7214	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7215	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
7216	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
7217
7218	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7219
7220	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7221	to start executing. */
7222
7223	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7224	/*
7225	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7226	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7227	*/
7228	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7229	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7230	{
7231	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7232	uint64_t u64HostTscAux = 0;
7233	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7234	AssertRC(rc2);
7235	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7236	}
7237	#endif
7238	}
7239
7240
7241	/**
7242	* Performs some essential restoration of state after running guest code in
7243	* VT-x.
7244	*
7245	* @param pVM Pointer to the VM.
7246	* @param pVCpu Pointer to the VMCPU.
7247	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7248	* out-of-sync. Make sure to update the required fields
7249	* before using them.
7250	* @param pVmxTransient Pointer to the VMX transient structure.
7251	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7252	*
7253	* @remarks Called with interrupts disabled.
7254	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7255	* unconditionally when it is safe to do so.
7256	*/
7257	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7258	{
7259	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7260
7261	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7262	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7263	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7264	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7265	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7266
7267	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7268	{
7269	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7270	/* Restore host's TSC_AUX. */
7271	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7272	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7273	#endif
7274	/** @todo Find a way to fix hardcoding a guestimate. */
7275	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7276	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7277	}
7278
7279	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7280	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7281	Assert(!(ASMGetFlags() & X86_EFL_IF));
7282	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7283
7284	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7285	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7286
7287	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7288	uint32_t uExitReason;
7289	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7290	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7291	AssertRC(rc);
7292	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7293	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7294
7295	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
7296	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7297
7298	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7299	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7300	{
7301	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7302	pVmxTransient->fVMEntryFailed));
7303	return;
7304	}
7305
7306	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7307	{
7308	/* Update the guest interruptibility-state from the VMCS. */
7309	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7310	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7311	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7312	AssertRC(rc);
7313	#endif
7314	/*
7315	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7316	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7317	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7318	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7319	*/
7320	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7321	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7322	{
7323	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7324	AssertRC(rc);
7325	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7326	}
7327	}
7328	}
7329
7330
7331
7332	/**
7333	* Runs the guest code using VT-x the normal way.
7334	*
7335	* @returns VBox status code.
7336	* @param pVM Pointer to the VM.
7337	* @param pVCpu Pointer to the VMCPU.
7338	* @param pCtx Pointer to the guest-CPU context.
7339	*
7340	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
7341	* @remarks Called with preemption disabled.
7342	*/
7343	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7344	{
7345	VMXTRANSIENT VmxTransient;
7346	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7347	int rc = VERR_INTERNAL_ERROR_5;
7348	uint32_t cLoops = 0;
7349
7350	for (;; cLoops++)
7351	{
7352	Assert(!HMR0SuspendPending());
7353	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7354	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7355	(unsigned)RTMpCpuId(), cLoops));
7356
7357	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7358	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7359	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7360	if (rc != VINF_SUCCESS)
7361	break;
7362
7363	/*
7364	* No longjmps to ring-3 from this point on!!!
7365	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7366	* This also disables flushing of the R0-logger instance (if any).
7367	*/
7368	VMMRZCallRing3Disable(pVCpu);
7369	VMMRZCallRing3RemoveNotification(pVCpu);
7370	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7371
7372	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7373	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7374
7375	/*
7376	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7377	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7378	*/
7379	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7380	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7381	{
7382	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7383	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7384	return rc;
7385	}
7386
7387	/* Handle the VM-exit. */
7388	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7389	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7390	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7391	HMVMX_START_EXIT_DISPATCH_PROF();
7392	#ifdef HMVMX_USE_FUNCTION_TABLE
7393	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7394	#else
7395	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7396	#endif
7397	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7398	if (rc != VINF_SUCCESS)
7399	break;
7400	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7401	{
7402	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7403	rc = VINF_EM_RAW_INTERRUPT;
7404	break;
7405	}
7406	}
7407
7408	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7409	return rc;
7410	}
7411
7412
7413	/**
7414	* Single steps guest code using VT-x.
7415	*
7416	* @returns VBox status code.
7417	* @param pVM Pointer to the VM.
7418	* @param pVCpu Pointer to the VMCPU.
7419	* @param pCtx Pointer to the guest-CPU context.
7420	*
7421	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7422	* @remarks Called with preemption disabled.
7423	*/
7424	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7425	{
7426	VMXTRANSIENT VmxTransient;
7427	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7428	int rc = VERR_INTERNAL_ERROR_5;
7429	uint32_t cLoops = 0;
7430	uint16_t uCsStart = pCtx->cs.Sel;
7431	uint64_t uRipStart = pCtx->rip;
7432
7433	for (;; cLoops++)
7434	{
7435	Assert(!HMR0SuspendPending());
7436	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7437	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7438	(unsigned)RTMpCpuId(), cLoops));
7439
7440	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7441	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7442	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7443	if (rc != VINF_SUCCESS)
7444	break;
7445
7446	/*
7447	* No longjmps to ring-3 from this point on!!!
7448	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7449	* This also disables flushing of the R0-logger instance (if any).
7450	*/
7451	VMMRZCallRing3Disable(pVCpu);
7452	VMMRZCallRing3RemoveNotification(pVCpu);
7453	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7454
7455	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7456	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7457
7458	/*
7459	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7460	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7461	*/
7462	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7463	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7464	{
7465	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7466	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7467	return rc;
7468	}
7469
7470	/* Handle the VM-exit. */
7471	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7472	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7473	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7474	HMVMX_START_EXIT_DISPATCH_PROF();
7475	#ifdef HMVMX_USE_FUNCTION_TABLE
7476	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7477	#else
7478	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7479	#endif
7480	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7481	if (rc != VINF_SUCCESS)
7482	break;
7483	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7484	{
7485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7486	rc = VINF_EM_RAW_INTERRUPT;
7487	break;
7488	}
7489
7490	/*
7491	* Did the RIP change, if so, consider it a single step.
7492	* Otherwise, make sure one of the TFs gets set.
7493	*/
7494	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7495	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7496	AssertRCReturn(rc2, rc2);
7497	if ( pCtx->rip != uRipStart
7498	\|\| pCtx->cs.Sel != uCsStart)
7499	{
7500	rc = VINF_EM_DBG_STEPPED;
7501	break;
7502	}
7503	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7504	}
7505
7506	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7507	return rc;
7508	}
7509
7510
7511	/**
7512	* Runs the guest code using VT-x.
7513	*
7514	* @returns VBox status code.
7515	* @param pVM Pointer to the VM.
7516	* @param pVCpu Pointer to the VMCPU.
7517	* @param pCtx Pointer to the guest-CPU context.
7518	*
7519	* @remarks Called with preemption disabled.
7520	*/
7521	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7522	{
7523	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7524	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7525
7526	int rc;
7527	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7528	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7529	else
7530	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7531
7532	if (rc == VERR_EM_INTERPRETER)
7533	rc = VINF_EM_RAW_EMULATE_INSTR;
7534	else if (rc == VINF_EM_RESET)
7535	rc = VINF_EM_TRIPLE_FAULT;
7536	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7537	return rc;
7538	}
7539
7540
7541	#ifndef HMVMX_USE_FUNCTION_TABLE
7542	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7543	{
7544	int rc;
7545	switch (rcReason)
7546	{
7547	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7548	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7549	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7550	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7551	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7552	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7553	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7554	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7555	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7556	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7557	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7558	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7559	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7560	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7561	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7562	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7563	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7564	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7565	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7566	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7567	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7568	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7569	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7570	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7571	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7572	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7573	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7574	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7575	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7576	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7577	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7578	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7579	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7580
7581	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7582	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7583	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7584	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7585	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7586	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7587	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7588	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7589	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7590
7591	case VMX_EXIT_VMCALL:
7592	case VMX_EXIT_VMCLEAR:
7593	case VMX_EXIT_VMLAUNCH:
7594	case VMX_EXIT_VMPTRLD:
7595	case VMX_EXIT_VMPTRST:
7596	case VMX_EXIT_VMREAD:
7597	case VMX_EXIT_VMRESUME:
7598	case VMX_EXIT_VMWRITE:
7599	case VMX_EXIT_VMXOFF:
7600	case VMX_EXIT_VMXON:
7601	case VMX_EXIT_INVEPT:
7602	case VMX_EXIT_INVVPID:
7603	case VMX_EXIT_VMFUNC:
7604	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7605	break;
7606	default:
7607	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7608	break;
7609	}
7610	return rc;
7611	}
7612	#endif
7613
7614	#ifdef DEBUG
7615	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7616	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7617	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7618
7619	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7620	do \
7621	{ \
7622	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7623	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7624	} while (0)
7625
7626	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7627	do { \
7628	AssertPtr(pVCpu); \
7629	AssertPtr(pMixedCtx); \
7630	AssertPtr(pVmxTransient); \
7631	Assert(pVmxTransient->fVMEntryFailed == false); \
7632	Assert(ASMIntAreEnabled()); \
7633	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7634	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7635	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7636	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7637	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7638	HMVMX_ASSERT_PREEMPT_CPUID(); \
7639	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7640	} while (0)
7641
7642	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7643	do { \
7644	Log4Func(("\n")); \
7645	} while(0)
7646	#else /* Release builds */
7647	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7648	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7649	#endif
7650
7651
7652	/**
7653	* Advances the guest RIP after reading it from the VMCS.
7654	*
7655	* @returns VBox status code.
7656	* @param pVCpu Pointer to the VMCPU.
7657	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7658	* out-of-sync. Make sure to update the required fields
7659	* before using them.
7660	* @param pVmxTransient Pointer to the VMX transient structure.
7661	*
7662	* @remarks No-long-jump zone!!!
7663	*/
7664	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7665	{
7666	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7667	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7668	AssertRCReturn(rc, rc);
7669
7670	pMixedCtx->rip += pVmxTransient->cbInstr;
7671	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7672	return rc;
7673	}
7674
7675
7676	/**
7677	* Tries to determine what part of the guest-state VT-x has deemed as invalid
7678	* and update error record fields accordingly.
7679	*
7680	* @return VMX_IGS_* return codes.
7681	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7682	* wrong with the guest state.
7683	*
7684	* @param pVM Pointer to the VM.
7685	* @param pVCpu Pointer to the VMCPU.
7686	* @param pCtx Pointer to the guest-CPU state.
7687	*/
7688	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7689	{
7690	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7691	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7692	uError = (err); \
7693	break; \
7694	} else do {} while (0)
7695	/* Duplicate of IEM_IS_CANONICAL(). */
7696	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7697
7698	int rc;
7699	uint64_t u64Val;
7700	uint32_t u32Val;
7701	uint32_t uError = VMX_IGS_ERROR;
7702	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7703
7704	do
7705	{
7706	/*
7707	* CR0.
7708	*/
7709	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7710	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7711	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7712	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7713	if (fUnrestrictedGuest)
7714	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
7715
7716	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7717	AssertRCBreak(rc);
7718	HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
7719	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
7720	if ( !fUnrestrictedGuest
7721	&& (u32Val & X86_CR0_PG)
7722	&& !(u32Val & X86_CR0_PE))
7723	{
7724	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7725	}
7726
7727	/*
7728	* CR4.
7729	*/
7730	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7731	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7732	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7733	AssertRCBreak(rc);
7734	HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
7735	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
7736
7737	/*
7738	* IA32_DEBUGCTL MSR.
7739	*/
7740	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7741	AssertRCBreak(rc);
7742	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7743	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
7744	{
7745	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7746	}
7747	uint64_t u64DebugCtlMsr = u64Val;
7748
7749	#ifdef VBOX_STRICT
7750	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
7751	AssertRCBreak(rc);
7752	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
7753	#endif
7754	bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
7755
7756	/*
7757	* RIP and RFLAGS.
7758	*/
7759	uint32_t u32Eflags;
7760	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7761	if (HMVMX_IS_64BIT_HOST_MODE())
7762	{
7763	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
7764	AssertRCBreak(rc);
7765	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
7766	if ( !fLongModeGuest
7767	\|\| !pCtx->cs.Attr.n.u1Long)
7768	{
7769	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
7770	}
7771	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
7772	* must be identical if the "IA32e mode guest" VM-entry control is 1
7773	* and CS.L is 1. No check applies if the CPU supports 64
7774	* linear-address bits. */
7775
7776	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
7777	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7778	AssertRCBreak(rc);
7779	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
7780	VMX_IGS_RFLAGS_RESERVED);
7781	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7782	u32Eflags = u64Val;
7783	}
7784	else
7785	#endif
7786	{
7787	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
7788	AssertRCBreak(rc);
7789	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
7790	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7791	}
7792
7793	if ( fLongModeGuest
7794	\|\| !(pCtx->cr0 & X86_CR0_PE))
7795	{
7796	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
7797	}
7798
7799	uint32_t u32EntryInfo;
7800	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7801	AssertRCBreak(rc);
7802	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
7803	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7804	{
7805	HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
7806	}
7807
7808	/*
7809	* 64-bit checks.
7810	*/
7811	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7812	if (HMVMX_IS_64BIT_HOST_MODE())
7813	{
7814	if ( fLongModeGuest
7815	&& !fUnrestrictedGuest)
7816	{
7817	HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
7818	HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
7819	}
7820
7821	if ( !fLongModeGuest
7822	&& (pCtx->cr4 & X86_CR4_PCIDE))
7823	{
7824	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
7825	}
7826
7827	/** @todo CR3 field must be such that bits 63:52 and bits in the range
7828	* 51:32 beyond the processor's physical-address width are 0. */
7829
7830	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7831	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
7832	{
7833	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
7834	}
7835
7836	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
7837	AssertRCBreak(rc);
7838	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
7839
7840	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
7841	AssertRCBreak(rc);
7842	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
7843	}
7844	#endif
7845
7846	/*
7847	* PERF_GLOBAL MSR.
7848	*/
7849	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
7850	{
7851	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
7852	AssertRCBreak(rc);
7853	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
7854	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
7855	}
7856
7857	/*
7858	* PAT MSR.
7859	*/
7860	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
7861	{
7862	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
7863	AssertRCBreak(rc);
7864	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
7865	for (unsigned i = 0; i < 8; i++)
7866	{
7867	uint8_t u8Val = (u64Val & 0x7);
7868	if ( u8Val != 0 /* UC */
7869	\|\| u8Val != 1 /* WC */
7870	\|\| u8Val != 4 /* WT */
7871	\|\| u8Val != 5 /* WP */
7872	\|\| u8Val != 6 /* WB */
7873	\|\| u8Val != 7 /* UC- */)
7874	{
7875	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
7876	}
7877	u64Val >>= 3;
7878	}
7879	}
7880
7881	/*
7882	* EFER MSR.
7883	*/
7884	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
7885	{
7886	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
7887	AssertRCBreak(rc);
7888	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
7889	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
7890	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
7891	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
7892	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7893	\|\| (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
7894	}
7895
7896	/*
7897	* Segment registers.
7898	*/
7899	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7900	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
7901	if (!(u32Eflags & X86_EFL_VM))
7902	{
7903	/* CS */
7904	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
7905	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
7906	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
7907	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
7908	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7909	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
7910	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7911	/* CS cannot be loaded with NULL in protected mode. */
7912	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
7913	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
7914	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
7915	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
7916	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
7917	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
7918	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
7919	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
7920	else
7921	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
7922
7923	/* SS */
7924	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7925	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
7926	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
7927	if ( !(pCtx->cr0 & X86_CR0_PE)
7928	\|\| pCtx->cs.Attr.n.u4Type == 3)
7929	{
7930	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
7931	}
7932	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
7933	{
7934	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
7935	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
7936	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
7937	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
7938	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
7939	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7940	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
7941	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7942	}
7943
7944	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
7945	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
7946	{
7947	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
7948	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
7949	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7950	\|\| pCtx->ds.Attr.n.u4Type > 11
7951	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7952	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
7953	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
7954	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
7955	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7956	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
7957	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7958	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7959	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
7960	}
7961	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
7962	{
7963	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
7964	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
7965	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7966	\|\| pCtx->es.Attr.n.u4Type > 11
7967	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7968	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
7969	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
7970	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
7971	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7972	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
7973	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7974	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7975	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
7976	}
7977	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
7978	{
7979	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
7980	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
7981	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7982	\|\| pCtx->fs.Attr.n.u4Type > 11
7983	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
7984	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
7985	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
7986	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
7987	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7988	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
7989	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7990	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7991	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
7992	}
7993	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
7994	{
7995	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
7996	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
7997	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7998	\|\| pCtx->gs.Attr.n.u4Type > 11
7999	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8000	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8001	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8002	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8003	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8004	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8005	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8006	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8007	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8008	}
8009	/* 64-bit capable CPUs. */
8010	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8011	if (HMVMX_IS_64BIT_HOST_MODE())
8012	{
8013	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8014	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8015	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8016	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8017	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8018	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8019	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8020	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8021	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8022	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8023	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8024	}
8025	#endif
8026	}
8027	else
8028	{
8029	/* V86 mode checks. */
8030	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8031	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8032	{
8033	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8034	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8035	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8036	}
8037	else
8038	{
8039	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8040	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8041	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8042	}
8043
8044	/* CS */
8045	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8046	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8047	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8048	/* SS */
8049	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8050	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8051	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8052	/* DS */
8053	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8054	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8055	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8056	/* ES */
8057	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8058	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8059	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8060	/* FS */
8061	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8062	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8063	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8064	/* GS */
8065	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8066	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8067	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8068	/* 64-bit capable CPUs. */
8069	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8070	if (HMVMX_IS_64BIT_HOST_MODE())
8071	{
8072	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8073	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8074	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8075	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8076	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8077	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8078	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8079	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8080	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8081	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8082	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8083	}
8084	#endif
8085	}
8086
8087	/*
8088	* TR.
8089	*/
8090	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8091	/* 64-bit capable CPUs. */
8092	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8093	if (HMVMX_IS_64BIT_HOST_MODE())
8094	{
8095	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8096	}
8097	#endif
8098	if (fLongModeGuest)
8099	{
8100	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8101	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8102	}
8103	else
8104	{
8105	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8106	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8107	VMX_IGS_TR_ATTR_TYPE_INVALID);
8108	}
8109	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8110	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8111	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8112	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8113	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8114	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8115	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8116	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8117
8118	/*
8119	* GDTR and IDTR.
8120	*/
8121	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8122	if (HMVMX_IS_64BIT_HOST_MODE())
8123	{
8124	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8125	AssertRCBreak(rc);
8126	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8127
8128	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8129	AssertRCBreak(rc);
8130	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8131	}
8132	#endif
8133
8134	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8135	AssertRCBreak(rc);
8136	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8137
8138	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8139	AssertRCBreak(rc);
8140	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8141
8142	/*
8143	* Guest Non-Register State.
8144	*/
8145	/* Activity State. */
8146	uint32_t u32ActivityState;
8147	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8148	AssertRCBreak(rc);
8149	HMVMX_CHECK_BREAK( !u32ActivityState
8150	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.msr.vmx_misc)),
8151	VMX_IGS_ACTIVITY_STATE_INVALID);
8152	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8153	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8154	uint32_t u32IntrState;
8155	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8156	AssertRCBreak(rc);
8157	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8158	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8159	{
8160	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8161	}
8162
8163	/** @todo Activity state and injecting interrupts. Left as a todo since we
8164	* currently don't use activity states but ACTIVE. */
8165
8166	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8167	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8168
8169	/* Guest interruptibility-state. */
8170	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8171	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8172	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8173	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8174	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8175	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8176	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8177	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8178	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8179	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8180	{
8181	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8182	{
8183	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8184	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8185	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8186	}
8187	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8188	{
8189	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8190	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8191	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8192	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8193	}
8194	}
8195	/** @todo Assumes the processor is not in SMM. */
8196	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8197	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8198	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8199	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8200	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8201	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8202	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8203	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8204	{
8205	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8206	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8207	}
8208
8209	/* Pending debug exceptions. */
8210	if (HMVMX_IS_64BIT_HOST_MODE())
8211	{
8212	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8213	AssertRCBreak(rc);
8214	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8215	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8216	u32Val = u64Val; /* For pending debug exceptions checks below. */
8217	}
8218	else
8219	{
8220	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8221	AssertRCBreak(rc);
8222	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8223	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8224	}
8225
8226	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8227	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8228	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8229	{
8230	if ( (u32Eflags & X86_EFL_TF)
8231	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8232	{
8233	/* Bit 14 is PendingDebug.BS. */
8234	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8235	}
8236	if ( !(u32Eflags & X86_EFL_TF)
8237	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8238	{
8239	/* Bit 14 is PendingDebug.BS. */
8240	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8241	}
8242	}
8243
8244	/* VMCS link pointer. */
8245	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8246	AssertRCBreak(rc);
8247	if (u64Val != UINT64_C(0xffffffffffffffff))
8248	{
8249	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8250	/** @todo Bits beyond the processor's physical-address width MBZ. */
8251	/** @todo 32-bit located in memory referenced by value of this field (as a
8252	* physical address) must contain the processor's VMCS revision ID. */
8253	/** @todo SMM checks. */
8254	}
8255
8256	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8257
8258	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8259	if (uError == VMX_IGS_ERROR)
8260	uError = VMX_IGS_REASON_NOT_FOUND;
8261	} while (0);
8262
8263	pVCpu->hm.s.u32HMError = uError;
8264	return uError;
8265
8266	#undef HMVMX_ERROR_BREAK
8267	#undef HMVMX_CHECK_BREAK
8268	#undef HMVMX_IS_CANONICAL
8269	}
8270
8271	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8272	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8273	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8274
8275	/** @name VM-exit handlers.
8276	* @{
8277	*/
8278
8279	/**
8280	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8281	*/
8282	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8283	{
8284	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8285	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8286	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8287	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
8288	Assert(ASMIntAreEnabled());
8289	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8290	return VINF_SUCCESS;
8291	#endif
8292	return VINF_EM_RAW_INTERRUPT;
8293	}
8294
8295
8296	/**
8297	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8298	*/
8299	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8300	{
8301	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8302	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8303
8304	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8305	AssertRCReturn(rc, rc);
8306
8307	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8308	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8309	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8310	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8311
8312	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8313	{
8314	/*
8315	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8316	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8317	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8318	*
8319	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8320	*/
8321	VMXDispatchHostNmi();
8322	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
8323	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8324	return VINF_SUCCESS;
8325	}
8326
8327	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8328	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8329	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8330	{
8331	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8332	return VINF_SUCCESS;
8333	}
8334	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8335	{
8336	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8337	return rc;
8338	}
8339
8340	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8341	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8342	switch (uIntrType)
8343	{
8344	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8345	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8346	/* no break */
8347	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8348	{
8349	switch (uVector)
8350	{
8351	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8352	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8353	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8354	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8355	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8356	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8357	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8358	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8359	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8360	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8361	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8362	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8363	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8364	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8365	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8366	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8367	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8368	#endif
8369	default:
8370	{
8371	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8372	AssertRCReturn(rc, rc);
8373
8374	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8375	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8376	{
8377	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8378	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8379	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8380	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8381	AssertRCReturn(rc, rc);
8382	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8383	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8384	0 /* GCPtrFaultAddress */);
8385	AssertRCReturn(rc, rc);
8386	}
8387	else
8388	{
8389	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8390	pVCpu->hm.s.u32HMError = uVector;
8391	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8392	}
8393	break;
8394	}
8395	}
8396	break;
8397	}
8398
8399	default:
8400	{
8401	pVCpu->hm.s.u32HMError = uExitIntrInfo;
8402	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8403	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8404	break;
8405	}
8406	}
8407	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8408	return rc;
8409	}
8410
8411
8412	/**
8413	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8414	*/
8415	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8416	{
8417	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8418
8419	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8420	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8421	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8422	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8423	AssertRCReturn(rc, rc);
8424
8425	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8427	return VINF_SUCCESS;
8428	}
8429
8430
8431	/**
8432	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8433	*/
8434	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8435	{
8436	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8437	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8438	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8439	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8440	}
8441
8442
8443	/**
8444	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8445	*/
8446	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8447	{
8448	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8450	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8451	}
8452
8453
8454	/**
8455	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8456	*/
8457	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8458	{
8459	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8460	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8461	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8462	}
8463
8464
8465	/**
8466	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8467	*/
8468	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8469	{
8470	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8471	PVM pVM = pVCpu->CTX_SUFF(pVM);
8472	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8473	if (RT_LIKELY(rc == VINF_SUCCESS))
8474	{
8475	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8476	Assert(pVmxTransient->cbInstr == 2);
8477	}
8478	else
8479	{
8480	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8481	rc = VERR_EM_INTERPRETER;
8482	}
8483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8484	return rc;
8485	}
8486
8487
8488	/**
8489	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8490	*/
8491	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8492	{
8493	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8494	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8495	AssertRCReturn(rc, rc);
8496
8497	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8498	return VINF_EM_RAW_EMULATE_INSTR;
8499
8500	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8501	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8502	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8503	}
8504
8505
8506	/**
8507	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8508	*/
8509	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8510	{
8511	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8512	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8513	AssertRCReturn(rc, rc);
8514
8515	PVM pVM = pVCpu->CTX_SUFF(pVM);
8516	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8517	if (RT_LIKELY(rc == VINF_SUCCESS))
8518	{
8519	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8520	Assert(pVmxTransient->cbInstr == 2);
8521	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8522	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8523	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8524	}
8525	else
8526	{
8527	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8528	rc = VERR_EM_INTERPRETER;
8529	}
8530	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8531	return rc;
8532	}
8533
8534
8535	/**
8536	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8537	*/
8538	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8539	{
8540	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8541	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8542	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8543	AssertRCReturn(rc, rc);
8544
8545	PVM pVM = pVCpu->CTX_SUFF(pVM);
8546	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8547	if (RT_LIKELY(rc == VINF_SUCCESS))
8548	{
8549	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8550	Assert(pVmxTransient->cbInstr == 3);
8551	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8552	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8553	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8554	}
8555	else
8556	{
8557	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8558	rc = VERR_EM_INTERPRETER;
8559	}
8560	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8561	return rc;
8562	}
8563
8564
8565	/**
8566	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8567	*/
8568	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8569	{
8570	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8571	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8572	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8573	AssertRCReturn(rc, rc);
8574
8575	PVM pVM = pVCpu->CTX_SUFF(pVM);
8576	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8577	if (RT_LIKELY(rc == VINF_SUCCESS))
8578	{
8579	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8580	Assert(pVmxTransient->cbInstr == 2);
8581	}
8582	else
8583	{
8584	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8585	rc = VERR_EM_INTERPRETER;
8586	}
8587	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8588	return rc;
8589	}
8590
8591
8592	/**
8593	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8594	*/
8595	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8596	{
8597	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8598	PVM pVM = pVCpu->CTX_SUFF(pVM);
8599	Assert(!pVM->hm.s.fNestedPaging);
8600
8601	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8602	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8603	AssertRCReturn(rc, rc);
8604
8605	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8606	rc = VBOXSTRICTRC_VAL(rc2);
8607	if (RT_LIKELY(rc == VINF_SUCCESS))
8608	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8609	else
8610	{
8611	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8612	pVmxTransient->uExitQualification, rc));
8613	}
8614	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8615	return rc;
8616	}
8617
8618
8619	/**
8620	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8621	*/
8622	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8623	{
8624	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8625	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8626	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8627	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8628	AssertRCReturn(rc, rc);
8629
8630	PVM pVM = pVCpu->CTX_SUFF(pVM);
8631	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8632	if (RT_LIKELY(rc == VINF_SUCCESS))
8633	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8634	else
8635	{
8636	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8637	rc = VERR_EM_INTERPRETER;
8638	}
8639	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8640	return rc;
8641	}
8642
8643
8644	/**
8645	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8646	*/
8647	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8648	{
8649	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8650	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8651	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8652	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8653	AssertRCReturn(rc, rc);
8654
8655	PVM pVM = pVCpu->CTX_SUFF(pVM);
8656	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8657	rc = VBOXSTRICTRC_VAL(rc2);
8658	if (RT_LIKELY( rc == VINF_SUCCESS
8659	\|\| rc == VINF_EM_HALT))
8660	{
8661	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8662	AssertRCReturn(rc3, rc3);
8663
8664	if ( rc == VINF_EM_HALT
8665	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8666	{
8667	rc = VINF_SUCCESS;
8668	}
8669	}
8670	else
8671	{
8672	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8673	rc = VERR_EM_INTERPRETER;
8674	}
8675	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
8676	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8677	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8678	return rc;
8679	}
8680
8681
8682	/**
8683	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8684	*/
8685	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8686	{
8687	/*
8688	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8689	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8690	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
8691	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8692	*/
8693	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8694	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8695	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8696	}
8697
8698
8699	/**
8700	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8701	*/
8702	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8703	{
8704	/*
8705	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8706	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8707	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
8708	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8709	*/
8710	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8711	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8712	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8713	}
8714
8715
8716	/**
8717	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
8718	*/
8719	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8720	{
8721	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
8722	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8723	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
8724	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8725	}
8726
8727
8728	/**
8729	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
8730	*/
8731	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8732	{
8733	/*
8734	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
8735	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
8736	* See Intel spec. 25.3 "Other Causes of VM-exits".
8737	*/
8738	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8739	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
8740	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8741	}
8742
8743
8744	/**
8745	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
8746	* VM-exit.
8747	*/
8748	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8749	{
8750	/*
8751	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
8752	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
8753	*
8754	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
8755	* See Intel spec. "23.8 Restrictions on VMX operation".
8756	*/
8757	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8758	return VINF_SUCCESS;
8759	}
8760
8761
8762	/**
8763	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
8764	* VM-exit.
8765	*/
8766	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8767	{
8768	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8769	return VINF_EM_RESET;
8770	}
8771
8772
8773	/**
8774	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
8775	*/
8776	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8777	{
8778	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8779	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
8780	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8781	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8782	AssertRCReturn(rc, rc);
8783
8784	pMixedCtx->rip++;
8785	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8786	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
8787	rc = VINF_SUCCESS;
8788	else
8789	rc = VINF_EM_HALT;
8790
8791	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8792	return rc;
8793	}
8794
8795
8796	/**
8797	* VM-exit handler for instructions that result in a #UD exception delivered to
8798	* the guest.
8799	*/
8800	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8801	{
8802	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8803	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
8804	return VINF_SUCCESS;
8805	}
8806
8807
8808	/**
8809	* VM-exit handler for expiry of the VMX preemption timer.
8810	*/
8811	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8812	{
8813	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8814
8815	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
8816	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8817
8818	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
8819	PVM pVM = pVCpu->CTX_SUFF(pVM);
8820	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
8821	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
8822	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
8823	}
8824
8825
8826	/**
8827	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
8828	*/
8829	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8830	{
8831	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8832
8833	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
8834	/** @todo check if XSETBV is supported by the recompiler. */
8835	return VERR_EM_INTERPRETER;
8836	}
8837
8838
8839	/**
8840	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
8841	*/
8842	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8843	{
8844	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8845
8846	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
8847	/** @todo implement EMInterpretInvpcid() */
8848	return VERR_EM_INTERPRETER;
8849	}
8850
8851
8852	/**
8853	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
8854	* Error VM-exit.
8855	*/
8856	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8857	{
8858	uint32_t uIntrState;
8859	HMVMXHCUINTREG uHCReg;
8860	uint64_t u64Val;
8861	uint32_t u32Val;
8862
8863	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
8864	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
8865	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
8866	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
8867	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8868	AssertRCReturn(rc, rc);
8869
8870	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
8871	NOREF(uInvalidReason);
8872
8873	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
8874	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
8875	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
8876	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
8877
8878	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
8879	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
8880	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
8881	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
8882	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
8883	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
8884	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
8885	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
8886	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
8887	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
8888	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
8889	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
8890
8891	PVM pVM = pVCpu->CTX_SUFF(pVM);
8892	HMDumpRegs(pVM, pVCpu, pMixedCtx);
8893
8894	return VERR_VMX_INVALID_GUEST_STATE;
8895	}
8896
8897
8898	/**
8899	* VM-exit handler for VM-entry failure due to an MSR-load
8900	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
8901	*/
8902	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8903	{
8904	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8905	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8906	}
8907
8908
8909	/**
8910	* VM-exit handler for VM-entry failure due to a machine-check event
8911	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
8912	*/
8913	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8914	{
8915	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8916	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8917	}
8918
8919
8920	/**
8921	* VM-exit handler for all undefined reasons. Should never ever happen.. in
8922	* theory.
8923	*/
8924	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8925	{
8926	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
8927	return VERR_VMX_UNDEFINED_EXIT_CODE;
8928	}
8929
8930
8931	/**
8932	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
8933	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
8934	* Conditional VM-exit.
8935	*/
8936	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8937	{
8938	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8939
8940	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
8941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
8942	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
8943	return VERR_EM_INTERPRETER;
8944	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8945	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8946	}
8947
8948
8949	/**
8950	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
8951	*/
8952	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8953	{
8954	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8955
8956	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
8957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
8958	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
8959	return VERR_EM_INTERPRETER;
8960	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8961	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8962	}
8963
8964
8965	/**
8966	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
8967	*/
8968	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8969	{
8970	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8971
8972	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
8973	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8974	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8975	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8976	AssertRCReturn(rc, rc);
8977
8978	PVM pVM = pVCpu->CTX_SUFF(pVM);
8979	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8980	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
8981	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
8982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
8983
8984	if (RT_LIKELY(rc == VINF_SUCCESS))
8985	{
8986	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8987	Assert(pVmxTransient->cbInstr == 2);
8988	}
8989	return rc;
8990	}
8991
8992
8993	/**
8994	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
8995	*/
8996	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8997	{
8998	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8999	PVM pVM = pVCpu->CTX_SUFF(pVM);
9000	int rc = VINF_SUCCESS;
9001
9002	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9003	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9004	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9005	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9006	AssertRCReturn(rc, rc);
9007	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9008
9009	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9010	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9012
9013	if (RT_LIKELY(rc == VINF_SUCCESS))
9014	{
9015	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9016
9017	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9018	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9019	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9020	{
9021	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9022	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9023	EMInterpretWrmsr() changes it. */
9024	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9025	}
9026	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9027	{
9028	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9029	AssertRCReturn(rc, rc);
9030	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9031	}
9032	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9033	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9034
9035	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9036	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9037	{
9038	switch (pMixedCtx->ecx)
9039	{
9040	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9041	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9042	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9043	case MSR_K8_FS_BASE: /* no break */
9044	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9045	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9046	}
9047	}
9048	#ifdef VBOX_STRICT
9049	else
9050	{
9051	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9052	switch (pMixedCtx->ecx)
9053	{
9054	case MSR_IA32_SYSENTER_CS:
9055	case MSR_IA32_SYSENTER_EIP:
9056	case MSR_IA32_SYSENTER_ESP:
9057	case MSR_K8_FS_BASE:
9058	case MSR_K8_GS_BASE:
9059	{
9060	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9061	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9062	}
9063
9064	case MSR_K8_LSTAR:
9065	case MSR_K6_STAR:
9066	case MSR_K8_SF_MASK:
9067	case MSR_K8_TSC_AUX:
9068	case MSR_K8_KERNEL_GS_BASE:
9069	{
9070	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9071	pMixedCtx->ecx));
9072	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9073	}
9074	}
9075	}
9076	#endif /* VBOX_STRICT */
9077	}
9078	return rc;
9079	}
9080
9081
9082	/**
9083	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9084	*/
9085	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9086	{
9087	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9088
9089	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9091	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9092	return VERR_EM_INTERPRETER;
9093	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9094	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9095	}
9096
9097
9098	/**
9099	* VM-exit handler for when the TPR value is lowered below the specified
9100	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9101	*/
9102	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9103	{
9104	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9105	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9106
9107	/*
9108	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9109	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9110	* resume guest execution.
9111	*/
9112	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9113	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9114	return VINF_SUCCESS;
9115	}
9116
9117
9118	/**
9119	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9120	* VM-exit.
9121	*
9122	* @retval VINF_SUCCESS when guest execution can continue.
9123	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9124	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9125	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9126	* recompiler.
9127	*/
9128	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9129	{
9130	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9131	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9132	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9133	AssertRCReturn(rc, rc);
9134
9135	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9136	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9137	PVM pVM = pVCpu->CTX_SUFF(pVM);
9138	switch (uAccessType)
9139	{
9140	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9141	{
9142	#if 0
9143	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9144	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9145	#else
9146	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9147	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9148	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9149	#endif
9150	AssertRCReturn(rc, rc);
9151
9152	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9153	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9154	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9155	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9156
9157	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9158	{
9159	case 0: /* CR0 */
9160	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9161	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9162	break;
9163	case 2: /* C2 **/
9164	/* Nothing to do here, CR2 it's not part of the VMCS. */
9165	break;
9166	case 3: /* CR3 */
9167	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9168	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9169	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
9170	break;
9171	case 4: /* CR4 */
9172	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9173	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
9174	break;
9175	case 8: /* CR8 */
9176	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9177	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9178	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9179	break;
9180	default:
9181	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9182	break;
9183	}
9184
9185	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9186	break;
9187	}
9188
9189	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9190	{
9191	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9192	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9193	AssertRCReturn(rc, rc);
9194	Assert( !pVM->hm.s.fNestedPaging
9195	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9196	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9197
9198	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9199	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9200	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9201
9202	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9203	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9204	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9205	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9206	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9207	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9208	break;
9209	}
9210
9211	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9212	{
9213	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9214	AssertRCReturn(rc, rc);
9215	rc = EMInterpretCLTS(pVM, pVCpu);
9216	AssertRCReturn(rc, rc);
9217	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9218	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9219	Log4(("CRX CLTS write rc=%d\n", rc));
9220	break;
9221	}
9222
9223	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9224	{
9225	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9226	AssertRCReturn(rc, rc);
9227	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9228	if (RT_LIKELY(rc == VINF_SUCCESS))
9229	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9230	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9231	Log4(("CRX LMSW write rc=%d\n", rc));
9232	break;
9233	}
9234
9235	default:
9236	{
9237	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9238	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9239	}
9240	}
9241
9242	/* Validate possible error codes. */
9243	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9244	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9245	if (RT_SUCCESS(rc))
9246	{
9247	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9248	AssertRCReturn(rc2, rc2);
9249	}
9250
9251	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9252	return rc;
9253	}
9254
9255
9256	/**
9257	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9258	* VM-exit.
9259	*/
9260	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9261	{
9262	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9263	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9264
9265	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9266	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9267	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9268	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9269	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9270	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9271	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9272	AssertRCReturn(rc2, rc2);
9273
9274	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9275	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9276	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9277	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9278	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9279	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9280	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9281
9282	/* I/O operation lookup arrays. */
9283	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9284	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9285
9286	VBOXSTRICTRC rcStrict;
9287	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9288	const uint32_t cbInstr = pVmxTransient->cbInstr;
9289	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9290	PVM pVM = pVCpu->CTX_SUFF(pVM);
9291	if (fIOString)
9292	{
9293	/*
9294	* INS/OUTS - I/O String instruction.
9295	*
9296	* Use instruction-information if available, otherwise fall back on
9297	* interpreting the instruction.
9298	*/
9299	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9300	#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9301	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9302	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.msr.vmx_basic_info))
9303	{
9304	rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9305	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9306	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9307	AssertRCReturn(rc2, rc2);
9308	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9309	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9310	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9311	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9312	if (fIOWrite)
9313	{
9314	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9315	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9316	//if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9317	// hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9318	// pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9319	}
9320	else
9321	{
9322	AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9323	("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9324	VERR_HMVMX_IPE_4);
9325	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9326	//if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9327	// hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9328	}
9329	}
9330	else
9331	{
9332	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9333	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9334	AssertRCReturn(rc2, rc2);
9335	rcStrict = IEMExecOne(pVCpu);
9336	}
9337	/** @todo IEM needs to be setting these flags somehow. */
9338	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9339	fUpdateRipAlready = true;
9340
9341	#else
9342	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9343	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9344	if (RT_SUCCESS(rcStrict))
9345	{
9346	if (fIOWrite)
9347	{
9348	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9349	(DISCPUMODE)pDis->uAddrMode, cbValue);
9350	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9351	}
9352	else
9353	{
9354	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9355	(DISCPUMODE)pDis->uAddrMode, cbValue);
9356	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9357	}
9358	}
9359	else
9360	{
9361	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9362	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9363	}
9364	#endif
9365	}
9366	else
9367	{
9368	/*
9369	* IN/OUT - I/O instruction.
9370	*/
9371	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9372	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9373	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9374	if (fIOWrite)
9375	{
9376	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9377	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9378	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9380	}
9381	else
9382	{
9383	uint32_t u32Result = 0;
9384	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9385	if (IOM_SUCCESS(rcStrict))
9386	{
9387	/* Save result of I/O IN instr. in AL/AX/EAX. */
9388	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9389	}
9390	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9391	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9392	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9393	}
9394	}
9395
9396	if (IOM_SUCCESS(rcStrict))
9397	{
9398	if (!fUpdateRipAlready)
9399	{
9400	pMixedCtx->rip += cbInstr;
9401	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9402	}
9403
9404	/*
9405	* If any I/O breakpoints are armed, we need to check if one triggered
9406	* and take appropriate action.
9407	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9408	*/
9409	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9410	AssertRCReturn(rc2, rc2);
9411
9412	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9413	* execution engines about whether hyper BPs and such are pending. */
9414	uint32_t const uDr7 = pMixedCtx->dr[7];
9415	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9416	&& X86_DR7_ANY_RW_IO(uDr7)
9417	&& (pMixedCtx->cr4 & X86_CR4_DE))
9418	\|\| DBGFBpIsHwIoArmed(pVM)))
9419	{
9420	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9421	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9422
9423	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9424	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9425	{
9426	/* Raise #DB. */
9427	if (fIsGuestDbgActive)
9428	ASMSetDR6(pMixedCtx->dr[6]);
9429	if (pMixedCtx->dr[7] != uDr7)
9430	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9431
9432	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9433	}
9434	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9435	else if ( rcStrict2 != VINF_SUCCESS
9436	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9437	rcStrict = rcStrict2;
9438	}
9439	}
9440
9441	#ifdef DEBUG
9442	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9443	Assert(!fIOWrite);
9444	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9445	Assert(fIOWrite);
9446	else
9447	{
9448	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9449	* statuses, that the VMM device and some others may return. See
9450	* IOM_SUCCESS() for guidance. */
9451	AssertMsg( RT_FAILURE(rcStrict)
9452	\|\| rcStrict == VINF_SUCCESS
9453	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9454	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9455	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9456	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9457	}
9458	#endif
9459
9460	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9461	return VBOXSTRICTRC_TODO(rcStrict);
9462	}
9463
9464
9465	/**
9466	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9467	* VM-exit.
9468	*/
9469	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9470	{
9471	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9472
9473	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9474	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9475	AssertRCReturn(rc, rc);
9476	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9477	{
9478	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9479	AssertRCReturn(rc, rc);
9480	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9481	{
9482	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9483
9484	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9485	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9486	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9487	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9488	{
9489	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9490	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9491
9492	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9493	Assert(!pVCpu->hm.s.Event.fPending);
9494	pVCpu->hm.s.Event.fPending = true;
9495	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9496	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9497	AssertRCReturn(rc, rc);
9498	if (fErrorCodeValid)
9499	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9500	else
9501	pVCpu->hm.s.Event.u32ErrCode = 0;
9502	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9503	&& uVector == X86_XCPT_PF)
9504	{
9505	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9506	}
9507
9508	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9509	}
9510	}
9511	}
9512
9513	/** @todo Emulate task switch someday, currently just going back to ring-3 for
9514	* emulation. */
9515	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9516	return VERR_EM_INTERPRETER;
9517	}
9518
9519
9520	/**
9521	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9522	*/
9523	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9524	{
9525	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9526	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9527	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9528	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9529	AssertRCReturn(rc, rc);
9530	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9531	return VINF_EM_DBG_STEPPED;
9532	}
9533
9534
9535	/**
9536	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9537	*/
9538	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9539	{
9540	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9541
9542	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9543	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9544	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9545	return VINF_SUCCESS;
9546	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9547	return rc;
9548
9549	#if 0
9550	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9551	* just sync the whole thing. */
9552	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9553	#else
9554	/* Aggressive state sync. for now. */
9555	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9556	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9557	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9558	#endif
9559	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9560	AssertRCReturn(rc, rc);
9561
9562	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9563	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9564	switch (uAccessType)
9565	{
9566	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9567	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9568	{
9569	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9570	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9571	{
9572	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9573	}
9574
9575	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9576	GCPhys &= PAGE_BASE_GC_MASK;
9577	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9578	PVM pVM = pVCpu->CTX_SUFF(pVM);
9579	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9580	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9581
9582	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9583	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9584	CPUMCTX2CORE(pMixedCtx), GCPhys);
9585	rc = VBOXSTRICTRC_VAL(rc2);
9586	Log4(("ApicAccess rc=%d\n", rc));
9587	if ( rc == VINF_SUCCESS
9588	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9589	\|\| rc == VERR_PAGE_NOT_PRESENT)
9590	{
9591	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9592	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9593	rc = VINF_SUCCESS;
9594	}
9595	break;
9596	}
9597
9598	default:
9599	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9600	rc = VINF_EM_RAW_EMULATE_INSTR;
9601	break;
9602	}
9603
9604	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9605	return rc;
9606	}
9607
9608
9609	/**
9610	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9611	* VM-exit.
9612	*/
9613	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9614	{
9615	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9616
9617	/* We should -not- get this VM-exit if the guest is debugging. */
9618	if (CPUMIsGuestDebugStateActive(pVCpu))
9619	{
9620	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9621	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9622	}
9623
9624	int rc = VERR_INTERNAL_ERROR_5;
9625	if ( !DBGFIsStepping(pVCpu)
9626	&& !pVCpu->hm.s.fSingleInstruction
9627	&& !CPUMIsHyperDebugStateActive(pVCpu))
9628	{
9629	/* Don't intercept MOV DRx and #DB any more. */
9630	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9631	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9632	AssertRCReturn(rc, rc);
9633
9634	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9635	{
9636	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9637	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9638	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9639	AssertRCReturn(rc, rc);
9640	#endif
9641	}
9642
9643	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9644	PVM pVM = pVCpu->CTX_SUFF(pVM);
9645	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9646	Assert(CPUMIsGuestDebugStateActive(pVCpu));
9647
9648	#ifdef VBOX_WITH_STATISTICS
9649	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9650	AssertRCReturn(rc, rc);
9651	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9652	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9653	else
9654	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9655	#endif
9656	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9657	return VINF_SUCCESS;
9658	}
9659
9660	/*
9661	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9662	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9663	*/
9664	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9665	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9666	AssertRCReturn(rc, rc);
9667
9668	PVM pVM = pVCpu->CTX_SUFF(pVM);
9669	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9670	{
9671	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9672	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9673	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9674	if (RT_SUCCESS(rc))
9675	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9676	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9677	}
9678	else
9679	{
9680	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9681	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9682	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9683	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9684	}
9685
9686	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9687	if (RT_SUCCESS(rc))
9688	{
9689	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9690	AssertRCReturn(rc2, rc2);
9691	}
9692	return rc;
9693	}
9694
9695
9696	/**
9697	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9698	* Conditional VM-exit.
9699	*/
9700	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9701	{
9702	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9703	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9704
9705	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9706	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9707	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9708	return VINF_SUCCESS;
9709	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9710	return rc;
9711
9712	RTGCPHYS GCPhys = 0;
9713	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9714
9715	#if 0
9716	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9717	#else
9718	/* Aggressive state sync. for now. */
9719	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9720	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9721	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9722	#endif
9723	AssertRCReturn(rc, rc);
9724
9725	/*
9726	* If we succeed, resume guest execution.
9727	* If we fail in interpreting the instruction because we couldn't get the guest physical address
9728	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
9729	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
9730	* weird case. See @bugref{6043}.
9731	*/
9732	PVM pVM = pVCpu->CTX_SUFF(pVM);
9733	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
9734	rc = VBOXSTRICTRC_VAL(rc2);
9735	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
9736	if ( rc == VINF_SUCCESS
9737	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9738	\|\| rc == VERR_PAGE_NOT_PRESENT)
9739	{
9740	/* Successfully handled MMIO operation. */
9741	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9742	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9743	rc = VINF_SUCCESS;
9744	}
9745	return rc;
9746	}
9747
9748
9749	/**
9750	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
9751	* VM-exit.
9752	*/
9753	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9754	{
9755	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9756	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9757
9758	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9759	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9760	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9761	return VINF_SUCCESS;
9762	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9763	return rc;
9764
9765	RTGCPHYS GCPhys = 0;
9766	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9767	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9768	#if 0
9769	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9770	#else
9771	/* Aggressive state sync. for now. */
9772	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9773	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9774	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9775	#endif
9776	AssertRCReturn(rc, rc);
9777
9778	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
9779	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
9780
9781	RTGCUINT uErrorCode = 0;
9782	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
9783	uErrorCode \|= X86_TRAP_PF_ID;
9784	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
9785	uErrorCode \|= X86_TRAP_PF_RW;
9786	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
9787	uErrorCode \|= X86_TRAP_PF_P;
9788
9789	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
9790
9791	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
9792	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9793
9794	/* Handle the pagefault trap for the nested shadow table. */
9795	PVM pVM = pVCpu->CTX_SUFF(pVM);
9796	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
9797	TRPMResetTrap(pVCpu);
9798
9799	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
9800	if ( rc == VINF_SUCCESS
9801	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9802	\|\| rc == VERR_PAGE_NOT_PRESENT)
9803	{
9804	/* Successfully synced our nested page tables. */
9805	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
9806	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
9807	return VINF_SUCCESS;
9808	}
9809
9810	Log4(("EPT return to ring-3 rc=%d\n"));
9811	return rc;
9812	}
9813
9814	/** @} */
9815
9816	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9817	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
9818	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9819
9820	/** @name VM-exit exception handlers.
9821	* @{
9822	*/
9823
9824	/**
9825	* VM-exit exception handler for #MF (Math Fault: floating point exception).
9826	*/
9827	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9828	{
9829	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
9831
9832	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9833	AssertRCReturn(rc, rc);
9834
9835	if (!(pMixedCtx->cr0 & X86_CR0_NE))
9836	{
9837	/* Old-style FPU error reporting needs some extra work. */
9838	/** @todo don't fall back to the recompiler, but do it manually. */
9839	return VERR_EM_INTERPRETER;
9840	}
9841
9842	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9843	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9844	return rc;
9845	}
9846
9847
9848	/**
9849	* VM-exit exception handler for #BP (Breakpoint exception).
9850	*/
9851	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9852	{
9853	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9854	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
9855
9856	/** @todo Try optimize this by not saving the entire guest state unless
9857	* really needed. */
9858	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9859	AssertRCReturn(rc, rc);
9860
9861	PVM pVM = pVCpu->CTX_SUFF(pVM);
9862	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9863	if (rc == VINF_EM_RAW_GUEST_TRAP)
9864	{
9865	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9866	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9867	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9868	AssertRCReturn(rc, rc);
9869
9870	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9871	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9872	}
9873
9874	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
9875	return rc;
9876	}
9877
9878
9879	/**
9880	* VM-exit exception handler for #DB (Debug exception).
9881	*/
9882	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9883	{
9884	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9885	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
9886	Log6(("XcptDB\n"));
9887
9888	/*
9889	* Get the DR6-like values from the exit qualification and pass it to DBGF
9890	* for processing.
9891	*/
9892	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9893	AssertRCReturn(rc, rc);
9894
9895	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
9896	uint64_t uDR6 = X86_DR6_INIT_VAL;
9897	uDR6 \|= ( pVmxTransient->uExitQualification
9898	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
9899
9900	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
9901	if (rc == VINF_EM_RAW_GUEST_TRAP)
9902	{
9903	/*
9904	* The exception was for the guest. Update DR6, DR7.GD and
9905	* IA32_DEBUGCTL.LBR before forwarding it.
9906	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
9907	*/
9908	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
9909	pMixedCtx->dr[6] \|= uDR6;
9910	if (CPUMIsGuestDebugStateActive(pVCpu))
9911	ASMSetDR6(pMixedCtx->dr[6]);
9912
9913	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9914	AssertRCReturn(rc, rc);
9915
9916	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
9917	pMixedCtx->dr[7] &= ~X86_DR7_GD;
9918
9919	/* Paranoia. */
9920	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
9921	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
9922
9923	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
9924	AssertRCReturn(rc, rc);
9925
9926	/*
9927	* Raise #DB in the guest.
9928	*/
9929	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9930	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9931	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9932	AssertRCReturn(rc, rc);
9933	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9934	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9935	return VINF_SUCCESS;
9936	}
9937
9938	/*
9939	* Not a guest trap, must be a hypervisor related debug event then.
9940	* Update DR6 in case someone is interested in it.
9941	*/
9942	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
9943	AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
9944	CPUMSetHyperDR6(pVCpu, uDR6);
9945
9946	return rc;
9947	}
9948
9949
9950	/**
9951	* VM-exit exception handler for #NM (Device-not-available exception: floating
9952	* point exception).
9953	*/
9954	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9955	{
9956	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9957
9958	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9959	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
9960	#endif
9961
9962	/* We require CR0 and EFER. EFER is always up-to-date. */
9963	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9964	AssertRCReturn(rc, rc);
9965
9966	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
9967	PVM pVM = pVCpu->CTX_SUFF(pVM);
9968	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
9969	if (rc == VINF_SUCCESS)
9970	{
9971	Assert(CPUMIsGuestFPUStateActive(pVCpu));
9972	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9973	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
9974	return VINF_SUCCESS;
9975	}
9976
9977	/* Forward #NM to the guest. */
9978	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
9979	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9980	AssertRCReturn(rc, rc);
9981	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9982	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
9983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
9984	return rc;
9985	}
9986
9987
9988	/**
9989	* VM-exit exception handler for #GP (General-protection exception).
9990	*
9991	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
9992	*/
9993	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9994	{
9995	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9996	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
9997
9998	int rc = VERR_INTERNAL_ERROR_5;
9999	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10000	{
10001	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10002	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10003	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10004	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10005	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10006	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10007	AssertRCReturn(rc, rc);
10008	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10009	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10010	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10011	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10012	return rc;
10013	#else
10014	/* We don't intercept #GP. */
10015	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10016	return VERR_VMX_UNEXPECTED_EXCEPTION;
10017	#endif
10018	}
10019
10020	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10021	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10022
10023	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10024	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10025	AssertRCReturn(rc, rc);
10026
10027	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10028	uint32_t cbOp = 0;
10029	PVM pVM = pVCpu->CTX_SUFF(pVM);
10030	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10031	if (RT_SUCCESS(rc))
10032	{
10033	rc = VINF_SUCCESS;
10034	Assert(cbOp == pDis->cbInstr);
10035	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10036	switch (pDis->pCurInstr->uOpcode)
10037	{
10038	case OP_CLI:
10039	{
10040	pMixedCtx->eflags.Bits.u1IF = 0;
10041	pMixedCtx->rip += pDis->cbInstr;
10042	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
10043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10044	break;
10045	}
10046
10047	case OP_STI:
10048	{
10049	pMixedCtx->eflags.Bits.u1IF = 1;
10050	pMixedCtx->rip += pDis->cbInstr;
10051	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10052	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10053	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
10054	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10055	break;
10056	}
10057
10058	case OP_HLT:
10059	{
10060	rc = VINF_EM_HALT;
10061	pMixedCtx->rip += pDis->cbInstr;
10062	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
10063	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10064	break;
10065	}
10066
10067	case OP_POPF:
10068	{
10069	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10070	uint32_t cbParm = 0;
10071	uint32_t uMask = 0;
10072	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10073	{
10074	cbParm = 4;
10075	uMask = 0xffffffff;
10076	}
10077	else
10078	{
10079	cbParm = 2;
10080	uMask = 0xffff;
10081	}
10082
10083	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10084	RTGCPTR GCPtrStack = 0;
10085	X86EFLAGS Eflags;
10086	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10087	&GCPtrStack);
10088	if (RT_SUCCESS(rc))
10089	{
10090	Assert(sizeof(Eflags.u32) >= cbParm);
10091	Eflags.u32 = 0;
10092	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10093	}
10094	if (RT_FAILURE(rc))
10095	{
10096	rc = VERR_EM_INTERPRETER;
10097	break;
10098	}
10099	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10100	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10101	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10102	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10103	pMixedCtx->eflags.Bits.u1RF = 0;
10104	pMixedCtx->esp += cbParm;
10105	pMixedCtx->esp &= uMask;
10106	pMixedCtx->rip += pDis->cbInstr;
10107	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
10108	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10109	break;
10110	}
10111
10112	case OP_PUSHF:
10113	{
10114	uint32_t cbParm = 0;
10115	uint32_t uMask = 0;
10116	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10117	{
10118	cbParm = 4;
10119	uMask = 0xffffffff;
10120	}
10121	else
10122	{
10123	cbParm = 2;
10124	uMask = 0xffff;
10125	}
10126
10127	/* Get the stack pointer & push the contents of eflags onto the stack. */
10128	RTGCPTR GCPtrStack = 0;
10129	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10130	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10131	if (RT_FAILURE(rc))
10132	{
10133	rc = VERR_EM_INTERPRETER;
10134	break;
10135	}
10136	X86EFLAGS Eflags = pMixedCtx->eflags;
10137	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10138	Eflags.Bits.u1RF = 0;
10139	Eflags.Bits.u1VM = 0;
10140
10141	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10142	if (RT_FAILURE(rc))
10143	{
10144	rc = VERR_EM_INTERPRETER;
10145	break;
10146	}
10147	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10148	pMixedCtx->esp -= cbParm;
10149	pMixedCtx->esp &= uMask;
10150	pMixedCtx->rip += pDis->cbInstr;
10151	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
10152	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10153	break;
10154	}
10155
10156	case OP_IRET:
10157	{
10158	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10159	* instruction reference. */
10160	RTGCPTR GCPtrStack = 0;
10161	uint32_t uMask = 0xffff;
10162	uint16_t aIretFrame[3];
10163	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10164	{
10165	rc = VERR_EM_INTERPRETER;
10166	break;
10167	}
10168	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10169	&GCPtrStack);
10170	if (RT_SUCCESS(rc))
10171	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10172	if (RT_FAILURE(rc))
10173	{
10174	rc = VERR_EM_INTERPRETER;
10175	break;
10176	}
10177	pMixedCtx->eip = 0;
10178	pMixedCtx->ip = aIretFrame[0];
10179	pMixedCtx->cs.Sel = aIretFrame[1];
10180	pMixedCtx->cs.ValidSel = aIretFrame[1];
10181	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10182	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10183	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10184	pMixedCtx->sp += sizeof(aIretFrame);
10185	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
10186	\| HM_CHANGED_GUEST_RFLAGS;
10187	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10189	break;
10190	}
10191
10192	case OP_INT:
10193	{
10194	uint16_t uVector = pDis->Param1.uValue & 0xff;
10195	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10196	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10197	break;
10198	}
10199
10200	case OP_INTO:
10201	{
10202	if (pMixedCtx->eflags.Bits.u1OF)
10203	{
10204	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10205	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10206	}
10207	break;
10208	}
10209
10210	default:
10211	{
10212	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10213	EMCODETYPE_SUPERVISOR);
10214	rc = VBOXSTRICTRC_VAL(rc2);
10215	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
10216	Log4(("#GP rc=%Rrc\n", rc));
10217	break;
10218	}
10219	}
10220	}
10221	else
10222	rc = VERR_EM_INTERPRETER;
10223
10224	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10225	("#GP Unexpected rc=%Rrc\n", rc));
10226	return rc;
10227	}
10228
10229
10230	/**
10231	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10232	* the exception reported in the VMX transient structure back into the VM.
10233	*
10234	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
10235	* up-to-date.
10236	*/
10237	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10238	{
10239	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10240
10241	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10242	hmR0VmxCheckExitDueToEventDelivery(). */
10243	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10244	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10245	AssertRCReturn(rc, rc);
10246	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10247
10248	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10249	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10250	return VINF_SUCCESS;
10251	}
10252
10253
10254	/**
10255	* VM-exit exception handler for #PF (Page-fault exception).
10256	*/
10257	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10258	{
10259	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10260	PVM pVM = pVCpu->CTX_SUFF(pVM);
10261	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10262	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10263	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10264	AssertRCReturn(rc, rc);
10265
10266	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10267	if (pVM->hm.s.fNestedPaging)
10268	{
10269	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10270	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10271	{
10272	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10273	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10274	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10275	}
10276	else
10277	{
10278	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10279	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10280	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10281	}
10282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10283	return rc;
10284	}
10285	#else
10286	Assert(!pVM->hm.s.fNestedPaging);
10287	#endif
10288
10289	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10290	AssertRCReturn(rc, rc);
10291
10292	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10293	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10294
10295	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10296	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10297	(RTGCPTR)pVmxTransient->uExitQualification);
10298
10299	Log4(("#PF: rc=%Rrc\n", rc));
10300	if (rc == VINF_SUCCESS)
10301	{
10302	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10303	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10304	* memory? We don't update the whole state here... */
10305	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
10306	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
10307	TRPMResetTrap(pVCpu);
10308	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10309	return rc;
10310	}
10311	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10312	{
10313	if (!pVmxTransient->fVectoringPF)
10314	{
10315	/* It's a guest page fault and needs to be reflected to the guest. */
10316	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10317	TRPMResetTrap(pVCpu);
10318	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10319	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10320	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10321	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10322	}
10323	else
10324	{
10325	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10326	TRPMResetTrap(pVCpu);
10327	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10328	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10329	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10330	}
10331
10332	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10333	return VINF_SUCCESS;
10334	}
10335
10336	TRPMResetTrap(pVCpu);
10337	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10338	return rc;
10339	}
10340
10341	/** @} */
10342

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 47802

Download in other formats: