HMVMXR0.cpp@ 47743

Last change on this file since 47743 was 47743, checked in by vboxsync, 11 years ago
adjusted my assertion.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 430.9 KB

Line
1	/* $Id: HMVMXR0.cpp 47743 2013-08-14 23:49:19Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#endif
45
46
47	/*******************************************************************************
48	* Defined Constants And Macros *
49	*******************************************************************************/
50	#if defined(RT_ARCH_AMD64)
51	# define HMVMX_IS_64BIT_HOST_MODE() (true)
52	typedef RTHCUINTREG HMVMXHCUINTREG;
53	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54	extern "C" uint32_t g_fVMXIs64bitHost;
55	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56	typedef uint64_t HMVMXHCUINTREG;
57	#else
58	# define HMVMX_IS_64BIT_HOST_MODE() (false)
59	typedef RTHCUINTREG HMVMXHCUINTREG;
60	#endif
61
62	/** Use the function table. */
63	#define HMVMX_USE_FUNCTION_TABLE
64
65	/** Determine which tagged-TLB flush handler to use. */
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71	/** @name Updated-guest-state flags.
72	* @{ */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113	/** @} */
114
115	/** @name
116	* Flags to skip redundant reads of some common VMCS fields that are not part of
117	* the guest-CPU state but are in the transient structure.
118	*/
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125	/** @} */
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86).
129	*
130	* We need to intercept all exceptions manually (except #PF). #NM is also
131	* handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
132	* intercepted even in real-mode if we have Nested Paging support.
133	*/
134	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
135	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
136	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
137	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
138	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
139	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
140	\| RT_BIT(X86_XCPT_XF))
141
142	/**
143	* Exception bitmap mask for all contributory exceptions.
144	*
145	* Page fault is deliberately excluded here as it's conditional as to whether
146	* it's contributory or benign. Page faults are handled separately.
147	*/
148	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
149	\| RT_BIT(X86_XCPT_DE))
150
151	/** Maximum VM-instruction error number. */
152	#define HMVMX_INSTR_ERROR_MAX 28
153
154	/** Profiling macro. */
155	#ifdef HM_PROFILE_EXIT_DISPATCH
156	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
157	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
158	#else
159	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
160	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
161	#endif
162
163
164	/*******************************************************************************
165	* Structures and Typedefs *
166	*******************************************************************************/
167	/**
168	* VMX transient state.
169	*
170	* A state structure for holding miscellaneous information across
171	* VMX non-root operation and restored after the transition.
172	*/
173	typedef struct VMXTRANSIENT
174	{
175	/** The host's rflags/eflags. */
176	RTCCUINTREG uEFlags;
177	#if HC_ARCH_BITS == 32
178	uint32_t u32Alignment0;
179	#endif
180	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
181	uint64_t u64LStarMsr;
182	/** The guest's TPR value used for TPR shadowing. */
183	uint8_t u8GuestTpr;
184	/** Alignment. */
185	uint8_t abAlignment0[7];
186
187	/** The basic VM-exit reason. */
188	uint16_t uExitReason;
189	/** Alignment. */
190	uint16_t u16Alignment0;
191	/** The VM-exit interruption error code. */
192	uint32_t uExitIntrErrorCode;
193	/** The VM-exit exit qualification. */
194	uint64_t uExitQualification;
195
196	/** The VM-exit interruption-information field. */
197	uint32_t uExitIntrInfo;
198	/** The VM-exit instruction-length field. */
199	uint32_t cbInstr;
200	/** The VM-exit instruction-information field. */
201	union
202	{
203	/** Plain unsigned int representation. */
204	uint32_t u;
205	/** INS and OUTS information. */
206	struct
207	{
208	uint32_t u6Reserved0 : 6;
209	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
210	uint32_t u3AddrSize : 3;
211	uint32_t u5Reserved1 : 5;
212	/** The segment register (X86_SREG_XXX). */
213	uint32_t iSegReg : 3;
214	uint32_t uReserved2 : 14;
215	} StrIo;
216	} ExitInstrInfo;
217	/** Whether the VM-entry failed or not. */
218	bool fVMEntryFailed;
219	/** Alignment. */
220	uint8_t abAlignment1[3];
221
222	/** The VM-entry interruption-information field. */
223	uint32_t uEntryIntrInfo;
224	/** The VM-entry exception error code field. */
225	uint32_t uEntryXcptErrorCode;
226	/** The VM-entry instruction length field. */
227	uint32_t cbEntryInstr;
228
229	/** IDT-vectoring information field. */
230	uint32_t uIdtVectoringInfo;
231	/** IDT-vectoring error code. */
232	uint32_t uIdtVectoringErrorCode;
233
234	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
235	uint32_t fVmcsFieldsRead;
236	/** Whether TSC-offsetting should be setup before VM-entry. */
237	bool fUpdateTscOffsettingAndPreemptTimer;
238	/** Whether the VM-exit was caused by a page-fault during delivery of a
239	* contributory exception or a page-fault. */
240	bool fVectoringPF;
241	} VMXTRANSIENT;
242	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
243	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
244	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
245	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
246	/** Pointer to VMX transient state. */
247	typedef VMXTRANSIENT *PVMXTRANSIENT;
248
249
250	/**
251	* MSR-bitmap read permissions.
252	*/
253	typedef enum VMXMSREXITREAD
254	{
255	/** Reading this MSR causes a VM-exit. */
256	VMXMSREXIT_INTERCEPT_READ = 0xb,
257	/** Reading this MSR does not cause a VM-exit. */
258	VMXMSREXIT_PASSTHRU_READ
259	} VMXMSREXITREAD;
260
261	/**
262	* MSR-bitmap write permissions.
263	*/
264	typedef enum VMXMSREXITWRITE
265	{
266	/** Writing to this MSR causes a VM-exit. */
267	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
268	/** Writing to this MSR does not cause a VM-exit. */
269	VMXMSREXIT_PASSTHRU_WRITE
270	} VMXMSREXITWRITE;
271
272	/**
273	* VM-exit handler.
274	*
275	* @returns VBox status code.
276	* @param pVCpu Pointer to the VMCPU.
277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
278	* out-of-sync. Make sure to update the required
279	* fields before using them.
280	* @param pVmxTransient Pointer to the VMX-transient structure.
281	*/
282	#ifndef HMVMX_USE_FUNCTION_TABLE
283	typedef int FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	#else
285	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	/** Pointer to VM-exit handler. */
287	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
288	#endif
289
290
291	/*******************************************************************************
292	* Internal Functions *
293	*******************************************************************************/
294	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
295	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
296	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
297	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
298	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
299	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
300	#endif
301	#ifndef HMVMX_USE_FUNCTION_TABLE
302	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
303	# define HMVMX_EXIT_DECL static int
304	#else
305	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
306	#endif
307
308	/** @name VM-exit handlers.
309	* @{
310	*/
311	static FNVMEXITHANDLER hmR0VmxExitXcptOrNmi;
312	static FNVMEXITHANDLER hmR0VmxExitExtInt;
313	static FNVMEXITHANDLER hmR0VmxExitTripleFault;
314	static FNVMEXITHANDLER hmR0VmxExitInitSignal;
315	static FNVMEXITHANDLER hmR0VmxExitSipi;
316	static FNVMEXITHANDLER hmR0VmxExitIoSmi;
317	static FNVMEXITHANDLER hmR0VmxExitSmi;
318	static FNVMEXITHANDLER hmR0VmxExitIntWindow;
319	static FNVMEXITHANDLER hmR0VmxExitNmiWindow;
320	static FNVMEXITHANDLER hmR0VmxExitTaskSwitch;
321	static FNVMEXITHANDLER hmR0VmxExitCpuid;
322	static FNVMEXITHANDLER hmR0VmxExitGetsec;
323	static FNVMEXITHANDLER hmR0VmxExitHlt;
324	static FNVMEXITHANDLER hmR0VmxExitInvd;
325	static FNVMEXITHANDLER hmR0VmxExitInvlpg;
326	static FNVMEXITHANDLER hmR0VmxExitRdpmc;
327	static FNVMEXITHANDLER hmR0VmxExitRdtsc;
328	static FNVMEXITHANDLER hmR0VmxExitRsm;
329	static FNVMEXITHANDLER hmR0VmxExitSetPendingXcptUD;
330	static FNVMEXITHANDLER hmR0VmxExitMovCRx;
331	static FNVMEXITHANDLER hmR0VmxExitMovDRx;
332	static FNVMEXITHANDLER hmR0VmxExitIoInstr;
333	static FNVMEXITHANDLER hmR0VmxExitRdmsr;
334	static FNVMEXITHANDLER hmR0VmxExitWrmsr;
335	static FNVMEXITHANDLER hmR0VmxExitErrInvalidGuestState;
336	static FNVMEXITHANDLER hmR0VmxExitErrMsrLoad;
337	static FNVMEXITHANDLER hmR0VmxExitErrUndefined;
338	static FNVMEXITHANDLER hmR0VmxExitMwait;
339	static FNVMEXITHANDLER hmR0VmxExitMtf;
340	static FNVMEXITHANDLER hmR0VmxExitMonitor;
341	static FNVMEXITHANDLER hmR0VmxExitPause;
342	static FNVMEXITHANDLER hmR0VmxExitErrMachineCheck;
343	static FNVMEXITHANDLER hmR0VmxExitTprBelowThreshold;
344	static FNVMEXITHANDLER hmR0VmxExitApicAccess;
345	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
346	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
347	static FNVMEXITHANDLER hmR0VmxExitEptViolation;
348	static FNVMEXITHANDLER hmR0VmxExitEptMisconfig;
349	static FNVMEXITHANDLER hmR0VmxExitRdtscp;
350	static FNVMEXITHANDLER hmR0VmxExitPreemptTimer;
351	static FNVMEXITHANDLER hmR0VmxExitWbinvd;
352	static FNVMEXITHANDLER hmR0VmxExitXsetbv;
353	static FNVMEXITHANDLER hmR0VmxExitRdrand;
354	static FNVMEXITHANDLER hmR0VmxExitInvpcid;
355	/** @} */
356
357	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
358	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
359	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
360	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
361	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
362	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
363	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
365
366	/*******************************************************************************
367	* Global Variables *
368	*******************************************************************************/
369	#ifdef HMVMX_USE_FUNCTION_TABLE
370
371	/**
372	* VMX_EXIT dispatch table.
373	*/
374	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
375	{
376	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
377	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
378	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
379	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
380	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
381	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
382	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
383	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
384	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
385	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
386	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
387	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
388	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
389	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
390	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
391	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
392	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
393	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
394	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
395	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
396	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
397	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
398	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
399	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
400	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
401	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
402	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
403	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
404	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
405	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
406	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
407	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
408	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
409	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
410	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
411	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
412	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
413	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
414	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
415	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
416	/* 40 UNDEFINED */ hmR0VmxExitPause,
417	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
418	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
419	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
420	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
421	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
422	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
423	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
424	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
425	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
426	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
427	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
428	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
429	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
430	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
431	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
432	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
433	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
434	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
435	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
436	};
437	#endif /* HMVMX_USE_FUNCTION_TABLE */
438
439	#ifdef VBOX_STRICT
440	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
441	{
442	/* 0 */ "(Not Used)",
443	/* 1 */ "VMCALL executed in VMX root operation.",
444	/* 2 */ "VMCLEAR with invalid physical address.",
445	/* 3 */ "VMCLEAR with VMXON pointer.",
446	/* 4 */ "VMLAUNCH with non-clear VMCS.",
447	/* 5 */ "VMRESUME with non-launched VMCS.",
448	/* 6 */ "VMRESUME after VMXOFF",
449	/* 7 */ "VM entry with invalid control fields.",
450	/* 8 */ "VM entry with invalid host state fields.",
451	/* 9 */ "VMPTRLD with invalid physical address.",
452	/* 10 */ "VMPTRLD with VMXON pointer.",
453	/* 11 */ "VMPTRLD with incorrect revision identifier.",
454	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
455	/* 13 */ "VMWRITE to read-only VMCS component.",
456	/* 14 */ "(Not Used)",
457	/* 15 */ "VMXON executed in VMX root operation.",
458	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
459	/* 17 */ "VM entry with non-launched executing VMCS.",
460	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
461	/* 19 */ "VMCALL with non-clear VMCS.",
462	/* 20 */ "VMCALL with invalid VM-exit control fields.",
463	/* 21 */ "(Not Used)",
464	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
465	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
466	/* 24 */ "VMCALL with invalid SMM-monitor features.",
467	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
468	/* 26 */ "VM entry with events blocked by MOV SS.",
469	/* 27 */ "(Not Used)",
470	/* 28 */ "Invalid operand to INVEPT/INVVPID."
471	};
472	#endif /* VBOX_STRICT */
473
474
475
476	/**
477	* Updates the VM's last error record. If there was a VMX instruction error,
478	* reads the error data from the VMCS and updates VCPU's last error record as
479	* well.
480	*
481	* @param pVM Pointer to the VM.
482	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
483	* VERR_VMX_UNABLE_TO_START_VM or
484	* VERR_VMX_INVALID_VMCS_FIELD).
485	* @param rc The error code.
486	*/
487	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
488	{
489	AssertPtr(pVM);
490	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
491	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
492	{
493	AssertPtrReturnVoid(pVCpu);
494	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
495	}
496	pVM->hm.s.lLastError = rc;
497	}
498
499
500	/**
501	* Reads the VM-entry interruption-information field from the VMCS into the VMX
502	* transient structure.
503	*
504	* @returns VBox status code.
505	* @param pVmxTransient Pointer to the VMX transient structure.
506	*
507	* @remarks No-long-jump zone!!!
508	*/
509	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
510	{
511	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
512	AssertRCReturn(rc, rc);
513	return VINF_SUCCESS;
514	}
515
516
517	/**
518	* Reads the VM-entry exception error code field from the VMCS into
519	* the VMX transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVmxTransient Pointer to the VMX transient structure.
523	*
524	* @remarks No-long-jump zone!!!
525	*/
526	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
527	{
528	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
529	AssertRCReturn(rc, rc);
530	return VINF_SUCCESS;
531	}
532
533
534	/**
535	* Reads the VM-entry exception error code field from the VMCS into
536	* the VMX transient structure.
537	*
538	* @returns VBox status code.
539	* @param pVCpu Pointer to the VMCPU.
540	* @param pVmxTransient Pointer to the VMX transient structure.
541	*
542	* @remarks No-long-jump zone!!!
543	*/
544	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
545	{
546	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
547	AssertRCReturn(rc, rc);
548	return VINF_SUCCESS;
549	}
550
551
552	/**
553	* Reads the VM-exit interruption-information field from the VMCS into the VMX
554	* transient structure.
555	*
556	* @returns VBox status code.
557	* @param pVCpu Pointer to the VMCPU.
558	* @param pVmxTransient Pointer to the VMX transient structure.
559	*/
560	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
561	{
562	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
563	{
564	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
565	AssertRCReturn(rc, rc);
566	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
567	}
568	return VINF_SUCCESS;
569	}
570
571
572	/**
573	* Reads the VM-exit interruption error code from the VMCS into the VMX
574	* transient structure.
575	*
576	* @returns VBox status code.
577	* @param pVCpu Pointer to the VMCPU.
578	* @param pVmxTransient Pointer to the VMX transient structure.
579	*/
580	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
581	{
582	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
583	{
584	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
585	AssertRCReturn(rc, rc);
586	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
587	}
588	return VINF_SUCCESS;
589	}
590
591
592	/**
593	* Reads the VM-exit instruction length field from the VMCS into the VMX
594	* transient structure.
595	*
596	* @returns VBox status code.
597	* @param pVCpu Pointer to the VMCPU.
598	* @param pVmxTransient Pointer to the VMX transient structure.
599	*/
600	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
601	{
602	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
603	{
604	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
605	AssertRCReturn(rc, rc);
606	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
607	}
608	return VINF_SUCCESS;
609	}
610
611
612	/**
613	* Reads the VM-exit instruction-information field from the VMCS into
614	* the VMX transient structure.
615	*
616	* @returns VBox status code.
617	* @param pVCpu The cross context per CPU structure.
618	* @param pVmxTransient Pointer to the VMX transient structure.
619	*/
620	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
621	{
622	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
623	{
624	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
625	AssertRCReturn(rc, rc);
626	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
627	}
628	return VINF_SUCCESS;
629	}
630
631
632	/**
633	* Reads the exit qualification from the VMCS into the VMX transient structure.
634	*
635	* @returns VBox status code.
636	* @param pVCpu Pointer to the VMCPU.
637	* @param pVmxTransient Pointer to the VMX transient structure.
638	*/
639	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
640	{
641	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
642	{
643	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
644	AssertRCReturn(rc, rc);
645	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
646	}
647	return VINF_SUCCESS;
648	}
649
650
651	/**
652	* Reads the IDT-vectoring information field from the VMCS into the VMX
653	* transient structure.
654	*
655	* @returns VBox status code.
656	* @param pVmxTransient Pointer to the VMX transient structure.
657	*
658	* @remarks No-long-jump zone!!!
659	*/
660	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
661	{
662	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
663	{
664	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
665	AssertRCReturn(rc, rc);
666	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
667	}
668	return VINF_SUCCESS;
669	}
670
671
672	/**
673	* Reads the IDT-vectoring error code from the VMCS into the VMX
674	* transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVmxTransient Pointer to the VMX transient structure.
678	*/
679	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
680	{
681	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
682	{
683	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
684	AssertRCReturn(rc, rc);
685	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
686	}
687	return VINF_SUCCESS;
688	}
689
690
691	/**
692	* Enters VMX root mode operation on the current CPU.
693	*
694	* @returns VBox status code.
695	* @param pVM Pointer to the VM (optional, can be NULL, after
696	* a resume).
697	* @param HCPhysCpuPage Physical address of the VMXON region.
698	* @param pvCpuPage Pointer to the VMXON region.
699	*/
700	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
701	{
702	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
703	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
704	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
705
706	if (pVM)
707	{
708	/* Write the VMCS revision dword to the VMXON region. */
709	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
710	}
711
712	/* Enable the VMX bit in CR4 if necessary. */
713	RTCCUINTREG uCr4 = ASMGetCR4();
714	if (!(uCr4 & X86_CR4_VMXE))
715	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
716
717	/* Enter VMX root mode. */
718	int rc = VMXEnable(HCPhysCpuPage);
719	if (RT_FAILURE(rc))
720	ASMSetCR4(uCr4);
721
722	return rc;
723	}
724
725
726	/**
727	* Exits VMX root mode operation on the current CPU.
728	*
729	* @returns VBox status code.
730	*/
731	static int hmR0VmxLeaveRootMode(void)
732	{
733	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
734
735	/* If we're for some reason not in VMX root mode, then don't leave it. */
736	RTCCUINTREG uHostCR4 = ASMGetCR4();
737	if (uHostCR4 & X86_CR4_VMXE)
738	{
739	/* Exit VMX root mode and clear the VMX bit in CR4. */
740	VMXDisable();
741	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
742	return VINF_SUCCESS;
743	}
744
745	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
746	}
747
748
749	/**
750	* Allocates and maps one physically contiguous page. The allocated page is
751	* zero'd out. (Used by various VT-x structures).
752	*
753	* @returns IPRT status code.
754	* @param pMemObj Pointer to the ring-0 memory object.
755	* @param ppVirt Where to store the virtual address of the
756	* allocation.
757	* @param pPhys Where to store the physical address of the
758	* allocation.
759	*/
760	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
761	{
762	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
763	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
764	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
765
766	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
767	if (RT_FAILURE(rc))
768	return rc;
769	ppVirt = RTR0MemObjAddress(pMemObj);
770	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
771	ASMMemZero32(*ppVirt, PAGE_SIZE);
772	return VINF_SUCCESS;
773	}
774
775
776	/**
777	* Frees and unmaps an allocated physical page.
778	*
779	* @param pMemObj Pointer to the ring-0 memory object.
780	* @param ppVirt Where to re-initialize the virtual address of
781	* allocation as 0.
782	* @param pHCPhys Where to re-initialize the physical address of the
783	* allocation as 0.
784	*/
785	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
786	{
787	AssertPtr(pMemObj);
788	AssertPtr(ppVirt);
789	AssertPtr(pHCPhys);
790	if (*pMemObj != NIL_RTR0MEMOBJ)
791	{
792	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
793	AssertRC(rc);
794	*pMemObj = NIL_RTR0MEMOBJ;
795	*ppVirt = 0;
796	*pHCPhys = 0;
797	}
798	}
799
800
801	/**
802	* Worker function to free VT-x related structures.
803	*
804	* @returns IPRT status code.
805	* @param pVM Pointer to the VM.
806	*/
807	static void hmR0VmxStructsFree(PVM pVM)
808	{
809	for (VMCPUID i = 0; i < pVM->cCpus; i++)
810	{
811	PVMCPU pVCpu = &pVM->aCpus[i];
812	AssertPtr(pVCpu);
813
814	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
815	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
816	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
817	#endif
818
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
820	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
821
822	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
823	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
824	}
825
826	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
827	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
828	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
829	#endif
830	}
831
832
833	/**
834	* Worker function to allocate VT-x related VM structures.
835	*
836	* @returns IPRT status code.
837	* @param pVM Pointer to the VM.
838	*/
839	static int hmR0VmxStructsAlloc(PVM pVM)
840	{
841	/*
842	* Initialize members up-front so we can cleanup properly on allocation failure.
843	*/
844	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
845	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
846	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
847	pVM->hm.s.vmx.HCPhys##a_Name = 0;
848
849	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
850	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
851	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
852	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
853
854	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
855	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
856	#endif
857	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
858
859	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
860	for (VMCPUID i = 0; i < pVM->cCpus; i++)
861	{
862	PVMCPU pVCpu = &pVM->aCpus[i];
863	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
864	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
865	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
866	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
867	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
868	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
869	#endif
870	}
871	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
872	#undef VMXLOCAL_INIT_VM_MEMOBJ
873
874	/*
875	* Allocate all the VT-x structures.
876	*/
877	int rc = VINF_SUCCESS;
878	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
879	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
883	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
884	#endif
885
886	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
887	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
888	{
889	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
890	&pVM->hm.s.vmx.HCPhysApicAccess);
891	if (RT_FAILURE(rc))
892	goto cleanup;
893	}
894
895	/*
896	* Initialize per-VCPU VT-x structures.
897	*/
898	for (VMCPUID i = 0; i < pVM->cCpus; i++)
899	{
900	PVMCPU pVCpu = &pVM->aCpus[i];
901	AssertPtr(pVCpu);
902
903	/* Allocate the VM control structure (VMCS). */
904	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
905	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
906	if (RT_FAILURE(rc))
907	goto cleanup;
908
909	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
910	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
911	{
912	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
913	&pVCpu->hm.s.vmx.HCPhysVirtApic);
914	if (RT_FAILURE(rc))
915	goto cleanup;
916	}
917
918	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
919	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
920	{
921	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
922	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
923	if (RT_FAILURE(rc))
924	goto cleanup;
925	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
926	}
927
928	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
929	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
930	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
931	if (RT_FAILURE(rc))
932	goto cleanup;
933
934	/* Allocate the VM-exit MSR-load page for the host MSRs. */
935	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
936	if (RT_FAILURE(rc))
937	goto cleanup;
938	#endif
939	}
940
941	return VINF_SUCCESS;
942
943	cleanup:
944	hmR0VmxStructsFree(pVM);
945	return rc;
946	}
947
948
949	/**
950	* Does global VT-x initialization (called during module initialization).
951	*
952	* @returns VBox status code.
953	*/
954	VMMR0DECL(int) VMXR0GlobalInit(void)
955	{
956	#ifdef HMVMX_USE_FUNCTION_TABLE
957	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
958	# ifdef VBOX_STRICT
959	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
960	Assert(g_apfnVMExitHandlers[i]);
961	# endif
962	#endif
963	return VINF_SUCCESS;
964	}
965
966
967	/**
968	* Does global VT-x termination (called during module termination).
969	*/
970	VMMR0DECL(void) VMXR0GlobalTerm()
971	{
972	/* Nothing to do currently. */
973	}
974
975
976	/**
977	* Sets up and activates VT-x on the current CPU.
978	*
979	* @returns VBox status code.
980	* @param pCpu Pointer to the global CPU info struct.
981	* @param pVM Pointer to the VM (can be NULL after a host resume
982	* operation).
983	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
984	* fEnabledByHost is true).
985	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
986	* @a fEnabledByHost is true).
987	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
988	* enable VT-x on the host.
989	*/
990	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
991	{
992	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
993	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
994
995	if (!fEnabledByHost)
996	{
997	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
998	if (RT_FAILURE(rc))
999	return rc;
1000	}
1001
1002	/*
1003	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
1004	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
1005	* each time while reusing a VPID after hitting the MaxASID limit once.
1006	*/
1007	if ( pVM
1008	&& pVM->hm.s.fNestedPaging)
1009	{
1010	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
1011	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
1012	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1013	pCpu->fFlushAsidBeforeUse = false;
1014	}
1015	else
1016	{
1017	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
1018	* without Nested Paging triggered this function) we still have the risk
1019	* of potentially running with stale TLB-entries from other hypervisors
1020	* when later we use a VM with NestedPaging. To fix this properly we will
1021	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
1022	* 'vmx_ept_vpid_caps' from it. Sigh. */
1023	pCpu->fFlushAsidBeforeUse = true;
1024	}
1025
1026	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1027	++pCpu->cTlbFlushes;
1028
1029	return VINF_SUCCESS;
1030	}
1031
1032
1033	/**
1034	* Deactivates VT-x on the current CPU.
1035	*
1036	* @returns VBox status code.
1037	* @param pCpu Pointer to the global CPU info struct.
1038	* @param pvCpuPage Pointer to the VMXON region.
1039	* @param HCPhysCpuPage Physical address of the VMXON region.
1040	*
1041	* @remarks This function should never be called when SUPR0EnableVTx() or
1042	* similar was used to enable VT-x on the host.
1043	*/
1044	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1045	{
1046	NOREF(pCpu);
1047	NOREF(pvCpuPage);
1048	NOREF(HCPhysCpuPage);
1049
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051	return hmR0VmxLeaveRootMode();
1052	}
1053
1054
1055	/**
1056	* Sets the permission bits for the specified MSR in the MSR bitmap.
1057	*
1058	* @param pVCpu Pointer to the VMCPU.
1059	* @param uMSR The MSR value.
1060	* @param enmRead Whether reading this MSR causes a VM-exit.
1061	* @param enmWrite Whether writing this MSR causes a VM-exit.
1062	*/
1063	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1064	{
1065	int32_t iBit;
1066	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1067
1068	/*
1069	* Layout:
1070	* 0x000 - 0x3ff - Low MSR read bits
1071	* 0x400 - 0x7ff - High MSR read bits
1072	* 0x800 - 0xbff - Low MSR write bits
1073	* 0xc00 - 0xfff - High MSR write bits
1074	*/
1075	if (uMsr <= 0x00001FFF)
1076	iBit = uMsr;
1077	else if ( uMsr >= 0xC0000000
1078	&& uMsr <= 0xC0001FFF)
1079	{
1080	iBit = (uMsr - 0xC0000000);
1081	pbMsrBitmap += 0x400;
1082	}
1083	else
1084	{
1085	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1086	return;
1087	}
1088
1089	Assert(iBit <= 0x1fff);
1090	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1091	ASMBitSet(pbMsrBitmap, iBit);
1092	else
1093	ASMBitClear(pbMsrBitmap, iBit);
1094
1095	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1096	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1097	else
1098	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1099	}
1100
1101
1102	/**
1103	* Flushes the TLB using EPT.
1104	*
1105	* @returns VBox status code.
1106	* @param pVM Pointer to the VM.
1107	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1108	* enmFlush).
1109	* @param enmFlush Type of flush.
1110	*/
1111	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1112	{
1113	AssertPtr(pVM);
1114	Assert(pVM->hm.s.fNestedPaging);
1115
1116	uint64_t descriptor[2];
1117	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1118	descriptor[0] = 0;
1119	else
1120	{
1121	Assert(pVCpu);
1122	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1123	}
1124	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1125
1126	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1127	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1128	rc));
1129	if ( RT_SUCCESS(rc)
1130	&& pVCpu)
1131	{
1132	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1133	}
1134	}
1135
1136
1137	/**
1138	* Flushes the TLB using VPID.
1139	*
1140	* @returns VBox status code.
1141	* @param pVM Pointer to the VM.
1142	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1143	* enmFlush).
1144	* @param enmFlush Type of flush.
1145	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1146	* on @a enmFlush).
1147	*/
1148	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1149	{
1150	AssertPtr(pVM);
1151	Assert(pVM->hm.s.vmx.fVpid);
1152
1153	uint64_t descriptor[2];
1154	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1155	{
1156	descriptor[0] = 0;
1157	descriptor[1] = 0;
1158	}
1159	else
1160	{
1161	AssertPtr(pVCpu);
1162	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1163	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1164	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1165	descriptor[1] = GCPtr;
1166	}
1167
1168	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1169	AssertMsg(rc == VINF_SUCCESS,
1170	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1171	if ( RT_SUCCESS(rc)
1172	&& pVCpu)
1173	{
1174	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1175	}
1176	}
1177
1178
1179	/**
1180	* Invalidates a guest page by guest virtual address. Only relevant for
1181	* EPT/VPID, otherwise there is nothing really to invalidate.
1182	*
1183	* @returns VBox status code.
1184	* @param pVM Pointer to the VM.
1185	* @param pVCpu Pointer to the VMCPU.
1186	* @param GCVirt Guest virtual address of the page to invalidate.
1187	*/
1188	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1189	{
1190	AssertPtr(pVM);
1191	AssertPtr(pVCpu);
1192	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1193
1194	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1195	if (!fFlushPending)
1196	{
1197	/*
1198	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1199	* See @bugref{6043} and @bugref{6177}.
1200	*
1201	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1202	* function maybe called in a loop with individual addresses.
1203	*/
1204	if (pVM->hm.s.vmx.fVpid)
1205	{
1206	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1207	{
1208	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1209	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1210	}
1211	else
1212	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1213	}
1214	else if (pVM->hm.s.fNestedPaging)
1215	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1216	}
1217
1218	return VINF_SUCCESS;
1219	}
1220
1221
1222	/**
1223	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1224	* otherwise there is nothing really to invalidate.
1225	*
1226	* @returns VBox status code.
1227	* @param pVM Pointer to the VM.
1228	* @param pVCpu Pointer to the VMCPU.
1229	* @param GCPhys Guest physical address of the page to invalidate.
1230	*/
1231	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1232	{
1233	LogFlowFunc(("%RGp\n", GCPhys));
1234
1235	/*
1236	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1237	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1238	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1239	*/
1240	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1241	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1242	return VINF_SUCCESS;
1243	}
1244
1245
1246	/**
1247	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1248	* case where neither EPT nor VPID is supported by the CPU.
1249	*
1250	* @param pVM Pointer to the VM.
1251	* @param pVCpu Pointer to the VMCPU.
1252	*
1253	* @remarks Called with interrupts disabled.
1254	*/
1255	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1256	{
1257	NOREF(pVM);
1258	AssertPtr(pVCpu);
1259	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1260	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1261
1262	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1263	AssertPtr(pCpu);
1264
1265	pVCpu->hm.s.TlbShootdown.cPages = 0;
1266	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1267	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1268	pVCpu->hm.s.fForceTLBFlush = false;
1269	return;
1270	}
1271
1272
1273	/**
1274	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1275	*
1276	* @param pVM Pointer to the VM.
1277	* @param pVCpu Pointer to the VMCPU.
1278	* @remarks All references to "ASID" in this function pertains to "VPID" in
1279	* Intel's nomenclature. The reason is, to avoid confusion in compare
1280	* statements since the host-CPU copies are named "ASID".
1281	*
1282	* @remarks Called with interrupts disabled.
1283	*/
1284	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1285	{
1286	#ifdef VBOX_WITH_STATISTICS
1287	bool fTlbFlushed = false;
1288	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1289	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1290	if (!fTlbFlushed) \
1291	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1292	} while (0)
1293	#else
1294	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1295	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1296	#endif
1297
1298	AssertPtr(pVM);
1299	AssertPtr(pVCpu);
1300	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1301	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1302	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1303
1304	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1305	AssertPtr(pCpu);
1306
1307	/*
1308	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1309	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1310	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1311	*/
1312	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1313	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1314	{
1315	++pCpu->uCurrentAsid;
1316	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1317	{
1318	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1319	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1320	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1321	}
1322
1323	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1324	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1325	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1326
1327	/*
1328	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1329	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1330	*/
1331	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1332	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1333	HMVMX_SET_TAGGED_TLB_FLUSHED();
1334	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1335	}
1336
1337	/* Check for explicit TLB shootdowns. */
1338	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1339	{
1340	/*
1341	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1342	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1343	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1344	* but not guest-physical mappings.
1345	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1346	*/
1347	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1348	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1349	HMVMX_SET_TAGGED_TLB_FLUSHED();
1350	}
1351
1352	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1353	* not be executed. See hmQueueInvlPage() where it is commented
1354	* out. Support individual entry flushing someday. */
1355	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1356	{
1357	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1358
1359	/*
1360	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1361	* as supported by the CPU.
1362	*/
1363	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1364	{
1365	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1366	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1367	}
1368	else
1369	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1370
1371	HMVMX_SET_TAGGED_TLB_FLUSHED();
1372	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1373	}
1374
1375	pVCpu->hm.s.TlbShootdown.cPages = 0;
1376	pVCpu->hm.s.fForceTLBFlush = false;
1377
1378	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1379
1380	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1381	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1382	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1383	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1384	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1385	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1386	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1387	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1388
1389	/* Update VMCS with the VPID. */
1390	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1391	AssertRC(rc);
1392
1393	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1394	}
1395
1396
1397	/**
1398	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1399	*
1400	* @returns VBox status code.
1401	* @param pVM Pointer to the VM.
1402	* @param pVCpu Pointer to the VMCPU.
1403	*
1404	* @remarks Called with interrupts disabled.
1405	*/
1406	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1407	{
1408	AssertPtr(pVM);
1409	AssertPtr(pVCpu);
1410	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1411	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1412
1413	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1414	AssertPtr(pCpu);
1415
1416	/*
1417	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1418	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1419	*/
1420	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1421	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1422	{
1423	pVCpu->hm.s.fForceTLBFlush = true;
1424	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1425	}
1426
1427	/* Check for explicit TLB shootdown flushes. */
1428	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1429	{
1430	pVCpu->hm.s.fForceTLBFlush = true;
1431	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1432	}
1433
1434	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1435	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1436
1437	if (pVCpu->hm.s.fForceTLBFlush)
1438	{
1439	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1440	pVCpu->hm.s.fForceTLBFlush = false;
1441	}
1442	else
1443	{
1444	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1445	* not be executed. See hmQueueInvlPage() where it is commented
1446	* out. Support individual entry flushing someday. */
1447	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1448	{
1449	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1450	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1451	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1452	}
1453	else
1454	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1455	}
1456
1457	pVCpu->hm.s.TlbShootdown.cPages = 0;
1458	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1459	}
1460
1461
1462	/**
1463	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1464	*
1465	* @returns VBox status code.
1466	* @param pVM Pointer to the VM.
1467	* @param pVCpu Pointer to the VMCPU.
1468	*
1469	* @remarks Called with interrupts disabled.
1470	*/
1471	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1472	{
1473	AssertPtr(pVM);
1474	AssertPtr(pVCpu);
1475	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1476	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1477
1478	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1479
1480	/*
1481	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1482	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1483	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1484	*/
1485	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1486	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1487	{
1488	pVCpu->hm.s.fForceTLBFlush = true;
1489	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1490	}
1491
1492	/* Check for explicit TLB shootdown flushes. */
1493	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1494	{
1495	/*
1496	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1497	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1498	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1499	*/
1500	pVCpu->hm.s.fForceTLBFlush = true;
1501	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1502	}
1503
1504	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1505	if (pVCpu->hm.s.fForceTLBFlush)
1506	{
1507	++pCpu->uCurrentAsid;
1508	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1509	{
1510	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1511	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1512	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1513	}
1514
1515	pVCpu->hm.s.fForceTLBFlush = false;
1516	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1517	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1518	if (pCpu->fFlushAsidBeforeUse)
1519	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1520	}
1521	else
1522	{
1523	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1524	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1525	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1526	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1527
1528	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1529	* not be executed. See hmQueueInvlPage() where it is commented
1530	* out. Support individual entry flushing someday. */
1531	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1532	{
1533	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1534	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1535	{
1536	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1537	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1538	}
1539	else
1540	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1541	}
1542	else
1543	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1544	}
1545
1546	pVCpu->hm.s.TlbShootdown.cPages = 0;
1547	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1548
1549	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1550	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1551	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1552	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1553	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1554	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1555
1556	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1557	AssertRC(rc);
1558	}
1559
1560
1561	/**
1562	* Flushes the guest TLB entry based on CPU capabilities.
1563	*
1564	* @param pVCpu Pointer to the VMCPU.
1565	*/
1566	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1567	{
1568	PVM pVM = pVCpu->CTX_SUFF(pVM);
1569	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1570	{
1571	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1572	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1573	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1574	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1575	default:
1576	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1577	break;
1578	}
1579	}
1580
1581
1582	/**
1583	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1584	* TLB entries from the host TLB before VM-entry.
1585	*
1586	* @returns VBox status code.
1587	* @param pVM Pointer to the VM.
1588	*/
1589	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1590	{
1591	/*
1592	* Determine optimal flush type for Nested Paging.
1593	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1594	* guest execution (see hmR3InitFinalizeR0()).
1595	*/
1596	if (pVM->hm.s.fNestedPaging)
1597	{
1598	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1599	{
1600	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1601	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1602	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1603	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1604	else
1605	{
1606	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1607	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1608	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1609	}
1610
1611	/* Make sure the write-back cacheable memory type for EPT is supported. */
1612	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1613	{
1614	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1615	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1616	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1617	}
1618	}
1619	else
1620	{
1621	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1622	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1623	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1624	}
1625	}
1626
1627	/*
1628	* Determine optimal flush type for VPID.
1629	*/
1630	if (pVM->hm.s.vmx.fVpid)
1631	{
1632	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1633	{
1634	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1635	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1636	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1637	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1638	else
1639	{
1640	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1641	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1642	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1643	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1644	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1645	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1646	pVM->hm.s.vmx.fVpid = false;
1647	}
1648	}
1649	else
1650	{
1651	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1652	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1653	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1654	pVM->hm.s.vmx.fVpid = false;
1655	}
1656	}
1657
1658	/*
1659	* Setup the handler for flushing tagged-TLBs.
1660	*/
1661	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1662	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1663	else if (pVM->hm.s.fNestedPaging)
1664	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1665	else if (pVM->hm.s.vmx.fVpid)
1666	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1667	else
1668	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1669	return VINF_SUCCESS;
1670	}
1671
1672
1673	/**
1674	* Sets up pin-based VM-execution controls in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVM Pointer to the VM.
1678	* @param pVCpu Pointer to the VMCPU.
1679	*/
1680	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1681	{
1682	AssertPtr(pVM);
1683	AssertPtr(pVCpu);
1684
1685	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1686	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1687
1688	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1689	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1690	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1691
1692	/* Enable the VMX preemption timer. */
1693	if (pVM->hm.s.vmx.fUsePreemptTimer)
1694	{
1695	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1696	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1697	}
1698
1699	if ((val & zap) != val)
1700	{
1701	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1702	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1703	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1704	}
1705
1706	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1707	AssertRCReturn(rc, rc);
1708
1709	/* Update VCPU with the currently set pin-based VM-execution controls. */
1710	pVCpu->hm.s.vmx.u32PinCtls = val;
1711	return rc;
1712	}
1713
1714
1715	/**
1716	* Sets up processor-based VM-execution controls in the VMCS.
1717	*
1718	* @returns VBox status code.
1719	* @param pVM Pointer to the VM.
1720	* @param pVMCPU Pointer to the VMCPU.
1721	*/
1722	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1723	{
1724	AssertPtr(pVM);
1725	AssertPtr(pVCpu);
1726
1727	int rc = VERR_INTERNAL_ERROR_5;
1728	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1729	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1730
1731	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1732	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1733	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1734	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1735	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1736	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1737	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1738
1739	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1740	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1741	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1742	{
1743	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1744	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1745	}
1746
1747	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1748	if (!pVM->hm.s.fNestedPaging)
1749	{
1750	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1751	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1752	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1753	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1754	}
1755
1756	/* Use TPR shadowing if supported by the CPU. */
1757	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1758	{
1759	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1760	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1761	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1762	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1763	AssertRCReturn(rc, rc);
1764
1765	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1766	/* CR8 writes causes a VM-exit based on TPR threshold. */
1767	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1768	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1769	}
1770	else
1771	{
1772	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1773	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1774	}
1775
1776	/* Use MSR-bitmaps if supported by the CPU. */
1777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1778	{
1779	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1780
1781	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1782	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1783	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1784	AssertRCReturn(rc, rc);
1785
1786	/*
1787	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1788	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1789	*/
1790	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1791	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1792	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1793	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1794	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1795	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1796	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1797	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1798	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1799	}
1800
1801	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1802	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1803	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1804
1805	if ((val & zap) != val)
1806	{
1807	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1808	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1809	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1810	}
1811
1812	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1813	AssertRCReturn(rc, rc);
1814
1815	/* Update VCPU with the currently set processor-based VM-execution controls. */
1816	pVCpu->hm.s.vmx.u32ProcCtls = val;
1817
1818	/*
1819	* Secondary processor-based VM-execution controls.
1820	*/
1821	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1822	{
1823	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1824	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1825
1826	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1827	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1828
1829	if (pVM->hm.s.fNestedPaging)
1830	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1831	else
1832	{
1833	/*
1834	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1835	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1836	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1837	*/
1838	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1839	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1840	}
1841
1842	if (pVM->hm.s.vmx.fVpid)
1843	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1844
1845	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1846	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1847
1848	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1849	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1850	* done dynamically. */
1851	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1852	{
1853	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1854	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1855	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1856	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1857	AssertRCReturn(rc, rc);
1858	}
1859
1860	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1861	{
1862	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1864	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1865	}
1866
1867	if ((val & zap) != val)
1868	{
1869	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1870	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1871	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1872	}
1873
1874	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1875	AssertRCReturn(rc, rc);
1876
1877	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1878	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1879	}
1880	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1881	{
1882	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1883	"available\n"));
1884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1885	}
1886
1887	return VINF_SUCCESS;
1888	}
1889
1890
1891	/**
1892	* Sets up miscellaneous (everything other than Pin & Processor-based
1893	* VM-execution) control fields in the VMCS.
1894	*
1895	* @returns VBox status code.
1896	* @param pVM Pointer to the VM.
1897	* @param pVCpu Pointer to the VMCPU.
1898	*/
1899	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1900	{
1901	AssertPtr(pVM);
1902	AssertPtr(pVCpu);
1903
1904	int rc = VERR_GENERAL_FAILURE;
1905
1906	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1907	#if 0
1908	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1909	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1910	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1911
1912	/*
1913	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1914	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1915	* We thus use the exception bitmap to control it rather than use both.
1916	*/
1917	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1918	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1919
1920	/** @todo Explore possibility of using IO-bitmaps. */
1921	/* All IO & IOIO instructions cause VM-exits. */
1922	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1923	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1924
1925	/* Initialize the MSR-bitmap area. */
1926	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1927	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1928	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1929	#endif
1930
1931	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1932	/* Setup MSR autoloading/storing. */
1933	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1934	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1935	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1936	AssertRCReturn(rc, rc);
1937	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1938	AssertRCReturn(rc, rc);
1939
1940	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1941	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1942	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1943	AssertRCReturn(rc, rc);
1944	#endif
1945
1946	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1947	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1948	AssertRCReturn(rc, rc);
1949
1950	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1951	#if 0
1952	/* Setup debug controls */
1953	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1954	AssertRCReturn(rc, rc);
1955	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1956	AssertRCReturn(rc, rc);
1957	#endif
1958
1959	return rc;
1960	}
1961
1962
1963	/**
1964	* Sets up the initial exception bitmap in the VMCS based on static conditions
1965	* (i.e. conditions that cannot ever change at runtime).
1966	*
1967	* @returns VBox status code.
1968	* @param pVM Pointer to the VM.
1969	* @param pVCpu Pointer to the VMCPU.
1970	*/
1971	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1972	{
1973	AssertPtr(pVM);
1974	AssertPtr(pVCpu);
1975
1976	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1977
1978	uint32_t u32XcptBitmap = 0;
1979
1980	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1981	if (!pVM->hm.s.fNestedPaging)
1982	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1983
1984	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1985	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1986	AssertRCReturn(rc, rc);
1987	return rc;
1988	}
1989
1990
1991	/**
1992	* Sets up the initial guest-state mask. The guest-state mask is consulted
1993	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1994	* for the nested virtualization case (as it would cause a VM-exit).
1995	*
1996	* @param pVCpu Pointer to the VMCPU.
1997	*/
1998	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1999	{
2000	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2001	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2002	return VINF_SUCCESS;
2003	}
2004
2005
2006	/**
2007	* Does per-VM VT-x initialization.
2008	*
2009	* @returns VBox status code.
2010	* @param pVM Pointer to the VM.
2011	*/
2012	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2013	{
2014	LogFlowFunc(("pVM=%p\n", pVM));
2015
2016	int rc = hmR0VmxStructsAlloc(pVM);
2017	if (RT_FAILURE(rc))
2018	{
2019	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2020	return rc;
2021	}
2022
2023	return VINF_SUCCESS;
2024	}
2025
2026
2027	/**
2028	* Does per-VM VT-x termination.
2029	*
2030	* @returns VBox status code.
2031	* @param pVM Pointer to the VM.
2032	*/
2033	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2034	{
2035	LogFlowFunc(("pVM=%p\n", pVM));
2036
2037	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2038	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2039	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2040	#endif
2041	hmR0VmxStructsFree(pVM);
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Sets up the VM for execution under VT-x.
2048	* This function is only called once per-VM during initialization.
2049	*
2050	* @returns VBox status code.
2051	* @param pVM Pointer to the VM.
2052	*/
2053	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2054	{
2055	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2056	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2057
2058	LogFlowFunc(("pVM=%p\n", pVM));
2059
2060	/*
2061	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2062	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2063	*/
2064	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2065	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2066	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2067	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2068	{
2069	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2070	return VERR_INTERNAL_ERROR;
2071	}
2072
2073	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2074	/*
2075	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2076	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2077	*/
2078	if ( pVM->hm.s.fAllow64BitGuests
2079	&& !HMVMX_IS_64BIT_HOST_MODE())
2080	{
2081	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2082	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2083	}
2084	#endif
2085
2086	/* Initialize these always, see hmR3InitFinalizeR0().*/
2087	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2088	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2089
2090	/* Setup the tagged-TLB flush handlers. */
2091	int rc = hmR0VmxSetupTaggedTlb(pVM);
2092	if (RT_FAILURE(rc))
2093	{
2094	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2095	return rc;
2096	}
2097
2098	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2099	{
2100	PVMCPU pVCpu = &pVM->aCpus[i];
2101	AssertPtr(pVCpu);
2102	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2103
2104	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2105	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2106
2107	/* Set revision dword at the beginning of the VMCS structure. */
2108	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2109
2110	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2111	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2112	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2113	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2114
2115	/* Load this VMCS as the current VMCS. */
2116	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2117	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2118	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2119
2120	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2121	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2122	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2123
2124	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2125	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2126	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2127
2128	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2129	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2130	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2131
2132	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2133	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2134	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2135
2136	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2137	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2138	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2139
2140	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2141	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2142	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2143	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2144	#endif
2145
2146	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2147	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2148	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2149	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2150
2151	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2152	}
2153
2154	return VINF_SUCCESS;
2155	}
2156
2157
2158	/**
2159	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2160	* the VMCS.
2161	*
2162	* @returns VBox status code.
2163	* @param pVM Pointer to the VM.
2164	* @param pVCpu Pointer to the VMCPU.
2165	*/
2166	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2167	{
2168	RTCCUINTREG uReg = ASMGetCR0();
2169	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2170	AssertRCReturn(rc, rc);
2171
2172	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2173	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2174	if (HMVMX_IS_64BIT_HOST_MODE())
2175	{
2176	uint64_t uRegCR3 = HMR0Get64bitCR3();
2177	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2178	}
2179	else
2180	#endif
2181	{
2182	uReg = ASMGetCR3();
2183	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2184	}
2185	AssertRCReturn(rc, rc);
2186
2187	uReg = ASMGetCR4();
2188	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2189	AssertRCReturn(rc, rc);
2190	return rc;
2191	}
2192
2193
2194	/**
2195	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2196	* the host-state area in the VMCS.
2197	*
2198	* @returns VBox status code.
2199	* @param pVM Pointer to the VM.
2200	* @param pVCpu Pointer to the VMCPU.
2201	*/
2202	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2203	{
2204	int rc = VERR_INTERNAL_ERROR_5;
2205	RTSEL uSelDS = 0;
2206	RTSEL uSelES = 0;
2207	RTSEL uSelFS = 0;
2208	RTSEL uSelGS = 0;
2209	RTSEL uSelTR = 0;
2210
2211	/*
2212	* Host DS, ES, FS and GS segment registers.
2213	*/
2214	#if HC_ARCH_BITS == 64
2215	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2216	uSelDS = ASMGetDS();
2217	uSelES = ASMGetES();
2218	uSelFS = ASMGetFS();
2219	uSelGS = ASMGetGS();
2220	#endif
2221
2222	/*
2223	* Host CS and SS segment registers.
2224	*/
2225	RTSEL uSelCS;
2226	RTSEL uSelSS;
2227	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2228	if (HMVMX_IS_64BIT_HOST_MODE())
2229	{
2230	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2231	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2232	}
2233	else
2234	{
2235	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2236	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2237	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2238	}
2239	#else
2240	uSelCS = ASMGetCS();
2241	uSelSS = ASMGetSS();
2242	#endif
2243
2244	/*
2245	* Host TR segment register.
2246	*/
2247	uSelTR = ASMGetTR();
2248
2249	#if HC_ARCH_BITS == 64
2250	/*
2251	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2252	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2253	*/
2254	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2255	{
2256	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2257	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2258	uSelDS = 0;
2259	}
2260	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2261	{
2262	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2263	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2264	uSelES = 0;
2265	}
2266	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2267	{
2268	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2269	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2270	uSelFS = 0;
2271	}
2272	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2273	{
2274	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2275	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2276	uSelGS = 0;
2277	}
2278	#endif
2279
2280	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2281	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2282	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2283	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2284	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2285	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2286	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2287	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2288	Assert(uSelCS);
2289	Assert(uSelTR);
2290
2291	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2292	#if 0
2293	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2294	Assert(uSelSS != 0);
2295	#endif
2296
2297	/* Write these host selector fields into the host-state area in the VMCS. */
2298	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2299	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2300	#if HC_ARCH_BITS == 64
2301	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2302	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2303	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2304	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2305	#endif
2306	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2307
2308	/*
2309	* Host GDTR and IDTR.
2310	*/
2311	RTGDTR Gdtr;
2312	RT_ZERO(Gdtr);
2313	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2314	if (HMVMX_IS_64BIT_HOST_MODE())
2315	{
2316	X86XDTR64 Gdtr64;
2317	X86XDTR64 Idtr64;
2318	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2319	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2320	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2321
2322	Gdtr.cbGdt = Gdtr64.cb;
2323	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2324	}
2325	else
2326	#endif
2327	{
2328	RTIDTR Idtr;
2329	ASMGetGDTR(&Gdtr);
2330	ASMGetIDTR(&Idtr);
2331	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2332	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2333
2334	#if HC_ARCH_BITS == 64
2335	/*
2336	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2337	* maximum limit (0xffff) on every VM-exit.
2338	*/
2339	if (Gdtr.cbGdt != 0xffff)
2340	{
2341	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2342	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2343	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2344	}
2345
2346	/*
2347	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2348	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2349	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2350	*/
2351	if (Idtr.cbIdt < 0x0fff)
2352	{
2353	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2354	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2355	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2356	}
2357	#endif
2358	}
2359
2360	/*
2361	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2362	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2363	*/
2364	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2365	{
2366	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2367	return VERR_VMX_INVALID_HOST_STATE;
2368	}
2369
2370	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2371	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2372	if (HMVMX_IS_64BIT_HOST_MODE())
2373	{
2374	/* We need the 64-bit TR base for hybrid darwin. */
2375	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2376	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2377	}
2378	else
2379	#endif
2380	{
2381	uintptr_t uTRBase;
2382	#if HC_ARCH_BITS == 64
2383	uTRBase = X86DESC64_BASE(pDesc);
2384
2385	/*
2386	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2387	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2388	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2389	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2390	*
2391	* [1] See Intel spec. 3.5 "System Descriptor Types".
2392	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2393	*/
2394	Assert(pDesc->System.u4Type == 11);
2395	if ( pDesc->System.u16LimitLow != 0x67
2396	\|\| pDesc->System.u4LimitHigh)
2397	{
2398	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2399	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2400
2401	/* Store the GDTR here as we need it while restoring TR. */
2402	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2403	}
2404	#else
2405	uTRBase = X86DESC_BASE(pDesc);
2406	#endif
2407	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2408	}
2409	AssertRCReturn(rc, rc);
2410
2411	/*
2412	* Host FS base and GS base.
2413	*/
2414	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2415	if (HMVMX_IS_64BIT_HOST_MODE())
2416	{
2417	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2418	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2419	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2420	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2421
2422	# if HC_ARCH_BITS == 64
2423	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2424	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2425	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2426	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2427	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2428	# endif
2429	}
2430	#endif
2431	return rc;
2432	}
2433
2434
2435	/**
2436	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2437	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2438	* the host after every successful VM exit.
2439	*
2440	* @returns VBox status code.
2441	* @param pVM Pointer to the VM.
2442	* @param pVCpu Pointer to the VMCPU.
2443	*/
2444	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2445	{
2446	AssertPtr(pVCpu);
2447	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2448
2449	int rc = VINF_SUCCESS;
2450	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2451	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2452	uint32_t cHostMsrs = 0;
2453	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2454
2455	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2456	{
2457	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2458
2459	# if HC_ARCH_BITS == 64
2460	/* Paranoia. 64-bit code requires these bits to be set always. */
2461	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2462
2463	/*
2464	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2465	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2466	* some reason (e.g. allow transparent reads) we would activate the code below.
2467	*/
2468	# if 0
2469	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2470	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2471	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2472	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2473	if (CPUMIsGuestInLongMode(pVCpu))
2474	{
2475	uint64_t u64GuestEfer;
2476	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2477	AssertRC(rc);
2478
2479	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2480	{
2481	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2482	pHostMsr->u32Reserved = 0;
2483	pHostMsr->u64Value = u64HostEfer;
2484	pHostMsr++; cHostMsrs++;
2485	}
2486	}
2487	# endif
2488	# else /* HC_ARCH_BITS != 64 */
2489	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2490	pHostMsr->u32Reserved = 0;
2491	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2492	if (CPUMIsGuestInLongMode(pVCpu))
2493	{
2494	/* Must match the EFER value in our 64 bits switcher. */
2495	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2496	}
2497	else
2498	# endif
2499	pHostMsr->u64Value = u64HostEfer;
2500	pHostMsr++; cHostMsrs++;
2501	# endif /* HC_ARCH_BITS == 64 */
2502	}
2503
2504	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2505	if (HMVMX_IS_64BIT_HOST_MODE())
2506	{
2507	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2508	pHostMsr->u32Reserved = 0;
2509	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2510	pHostMsr++; cHostMsrs++;
2511	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2512	pHostMsr->u32Reserved = 0;
2513	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2514	pHostMsr++; cHostMsrs++;
2515	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2516	pHostMsr->u32Reserved = 0;
2517	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2518	pHostMsr++; cHostMsrs++;
2519	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2520	pHostMsr->u32Reserved = 0;
2521	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2522	pHostMsr++; cHostMsrs++;
2523	}
2524	# endif
2525
2526	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2527	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2528	{
2529	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2530	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2531	}
2532
2533	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2534	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2535
2536	/*
2537	* Host Sysenter MSRs.
2538	*/
2539	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2540	AssertRCReturn(rc, rc);
2541	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2542	if (HMVMX_IS_64BIT_HOST_MODE())
2543	{
2544	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2545	AssertRCReturn(rc, rc);
2546	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2547	}
2548	else
2549	{
2550	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2551	AssertRCReturn(rc, rc);
2552	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2553	}
2554	#elif HC_ARCH_BITS == 32
2555	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2556	AssertRCReturn(rc, rc);
2557	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2558	#else
2559	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2560	AssertRCReturn(rc, rc);
2561	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2562	#endif
2563	AssertRCReturn(rc, rc);
2564
2565	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2566	* hmR0VmxSetupExitCtls() !! */
2567	return rc;
2568	}
2569
2570
2571	/**
2572	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2573	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2574	* controls".
2575	*
2576	* @returns VBox status code.
2577	* @param pVCpu Pointer to the VMCPU.
2578	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2579	* out-of-sync. Make sure to update the required fields
2580	* before using them.
2581	*
2582	* @remarks No-long-jump zone!!!
2583	*/
2584	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2585	{
2586	int rc = VINF_SUCCESS;
2587	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2588	{
2589	PVM pVM = pVCpu->CTX_SUFF(pVM);
2590	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2591	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2592
2593	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2594	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2595
2596	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2597	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2598	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2599	else
2600	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2601
2602	/*
2603	* The following should not be set (since we're not in SMM mode):
2604	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2605	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2606	*/
2607
2608	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2609	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2610	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2611
2612	if ((val & zap) != val)
2613	{
2614	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2615	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2616	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2617	}
2618
2619	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2620	AssertRCReturn(rc, rc);
2621
2622	/* Update VCPU with the currently set VM-exit controls. */
2623	pVCpu->hm.s.vmx.u32EntryCtls = val;
2624	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2625	}
2626	return rc;
2627	}
2628
2629
2630	/**
2631	* Sets up the VM-exit controls in the VMCS.
2632	*
2633	* @returns VBox status code.
2634	* @param pVM Pointer to the VM.
2635	* @param pVCpu Pointer to the VMCPU.
2636	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2637	* out-of-sync. Make sure to update the required fields
2638	* before using them.
2639	*
2640	* @remarks requires EFER.
2641	*/
2642	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2643	{
2644	int rc = VINF_SUCCESS;
2645	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2646	{
2647	PVM pVM = pVCpu->CTX_SUFF(pVM);
2648	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2649	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2650
2651	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2652	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2653
2654	/*
2655	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2656	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2657	*/
2658	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2659	if (HMVMX_IS_64BIT_HOST_MODE())
2660	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2661	else
2662	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2663	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2664	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2665	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2666	else
2667	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2668	#endif
2669
2670	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2671	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2672
2673	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2674	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2675	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2676	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2677	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2678
2679	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2680	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2681
2682	if ((val & zap) != val)
2683	{
2684	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2685	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2686	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2687	}
2688
2689	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2690	AssertRCReturn(rc, rc);
2691
2692	/* Update VCPU with the currently set VM-exit controls. */
2693	pVCpu->hm.s.vmx.u32ExitCtls = val;
2694	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2695	}
2696	return rc;
2697	}
2698
2699
2700	/**
2701	* Loads the guest APIC and related state.
2702	*
2703	* @returns VBox status code.
2704	* @param pVM Pointer to the VM.
2705	* @param pVCpu Pointer to the VMCPU.
2706	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2707	* out-of-sync. Make sure to update the required fields
2708	* before using them.
2709	*/
2710	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2711	{
2712	int rc = VINF_SUCCESS;
2713	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2714	{
2715	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2716	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2717	{
2718	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2719
2720	bool fPendingIntr = false;
2721	uint8_t u8Tpr = 0;
2722	uint8_t u8PendingIntr = 0;
2723	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2724	AssertRCReturn(rc, rc);
2725
2726	/*
2727	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2728	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2729	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2730	* the interrupt when we VM-exit for other reasons.
2731	*/
2732	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2733	uint32_t u32TprThreshold = 0;
2734	if (fPendingIntr)
2735	{
2736	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2737	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2738	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2739	if (u8PendingPriority <= u8TprPriority)
2740	u32TprThreshold = u8PendingPriority;
2741	else
2742	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2743	}
2744	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2745
2746	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2747	AssertRCReturn(rc, rc);
2748	}
2749
2750	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2751	}
2752	return rc;
2753	}
2754
2755
2756	/**
2757	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2758	*
2759	* @returns Guest's interruptibility-state.
2760	* @param pVCpu Pointer to the VMCPU.
2761	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2762	* out-of-sync. Make sure to update the required fields
2763	* before using them.
2764	*
2765	* @remarks No-long-jump zone!!!
2766	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2767	*/
2768	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2769	{
2770	/*
2771	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2772	* inhibit interrupts or clear any existing interrupt-inhibition.
2773	*/
2774	uint32_t uIntrState = 0;
2775	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2776	{
2777	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2778	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2779	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2780	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2781	{
2782	/*
2783	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2784	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2785	*/
2786	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2787	}
2788	else if (pMixedCtx->eflags.Bits.u1IF)
2789	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2790	else
2791	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2792	}
2793	return uIntrState;
2794	}
2795
2796
2797	/**
2798	* Loads the guest's interruptibility-state into the guest-state area in the
2799	* VMCS.
2800	*
2801	* @returns VBox status code.
2802	* @param pVCpu Pointer to the VMCPU.
2803	* @param uIntrState The interruptibility-state to set.
2804	*/
2805	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2806	{
2807	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2808	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2809	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2810	AssertRCReturn(rc, rc);
2811	return rc;
2812	}
2813
2814
2815	/**
2816	* Loads the guest's RIP into the guest-state area in the VMCS.
2817	*
2818	* @returns VBox status code.
2819	* @param pVCpu Pointer to the VMCPU.
2820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2821	* out-of-sync. Make sure to update the required fields
2822	* before using them.
2823	*
2824	* @remarks No-long-jump zone!!!
2825	*/
2826	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2827	{
2828	int rc = VINF_SUCCESS;
2829	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2830	{
2831	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2832	AssertRCReturn(rc, rc);
2833	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2834	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2835	}
2836	return rc;
2837	}
2838
2839
2840	/**
2841	* Loads the guest's RSP into the guest-state area in the VMCS.
2842	*
2843	* @returns VBox status code.
2844	* @param pVCpu Pointer to the VMCPU.
2845	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2846	* out-of-sync. Make sure to update the required fields
2847	* before using them.
2848	*
2849	* @remarks No-long-jump zone!!!
2850	*/
2851	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2852	{
2853	int rc = VINF_SUCCESS;
2854	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2855	{
2856	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2857	AssertRCReturn(rc, rc);
2858	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2859	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2860	}
2861	return rc;
2862	}
2863
2864
2865	/**
2866	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2867	*
2868	* @returns VBox status code.
2869	* @param pVCpu Pointer to the VMCPU.
2870	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2871	* out-of-sync. Make sure to update the required fields
2872	* before using them.
2873	*
2874	* @remarks No-long-jump zone!!!
2875	*/
2876	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2877	{
2878	int rc = VINF_SUCCESS;
2879	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2880	{
2881	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2882	Let us assert it as such and use 32-bit VMWRITE. */
2883	Assert(!(pMixedCtx->rflags.u64 >> 32));
2884	X86EFLAGS uEFlags = pMixedCtx->eflags;
2885	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2886	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2887
2888	/*
2889	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2890	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2891	*/
2892	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2893	{
2894	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2895	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2896	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2897	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2898	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2899	}
2900
2901	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2902	AssertRCReturn(rc, rc);
2903
2904	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2905	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2906	}
2907	return rc;
2908	}
2909
2910
2911	/**
2912	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2913	*
2914	* @returns VBox status code.
2915	* @param pVCpu Pointer to the VMCPU.
2916	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2917	* out-of-sync. Make sure to update the required fields
2918	* before using them.
2919	*
2920	* @remarks No-long-jump zone!!!
2921	*/
2922	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2923	{
2924	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2925	AssertRCReturn(rc, rc);
2926	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2927	AssertRCReturn(rc, rc);
2928	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2929	AssertRCReturn(rc, rc);
2930	return rc;
2931	}
2932
2933
2934	/**
2935	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2936	* in the VMCS.
2937	*
2938	* @returns VBox status code.
2939	* @param pVM Pointer to the VM.
2940	* @param pVCpu Pointer to the VMCPU.
2941	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2942	* out-of-sync. Make sure to update the required fields
2943	* before using them.
2944	*
2945	* @remarks No-long-jump zone!!!
2946	*/
2947	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2948	{
2949	int rc = VINF_SUCCESS;
2950	PVM pVM = pVCpu->CTX_SUFF(pVM);
2951
2952	/*
2953	* Guest CR0.
2954	* Guest FPU.
2955	*/
2956	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2957	{
2958	Assert(!(pCtx->cr0 >> 32));
2959	uint32_t u32GuestCR0 = pCtx->cr0;
2960
2961	/* The guest's view (read access) of its CR0 is unblemished. */
2962	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2963	AssertRCReturn(rc, rc);
2964	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2965
2966	/* Setup VT-x's view of the guest CR0. */
2967	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2968	if (pVM->hm.s.fNestedPaging)
2969	{
2970	if (CPUMIsGuestPagingEnabledEx(pCtx))
2971	{
2972	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2973	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2974	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2975	}
2976	else
2977	{
2978	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2979	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2980	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2981	}
2982
2983	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2984	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2985	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2986
2987	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2988	AssertRCReturn(rc, rc);
2989	}
2990	else
2991	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2992
2993	/*
2994	* Guest FPU bits.
2995	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2996	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2997	*/
2998	u32GuestCR0 \|= X86_CR0_NE;
2999	bool fInterceptNM = false;
3000	if (CPUMIsGuestFPUStateActive(pVCpu))
3001	{
3002	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3003	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3004	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3005	}
3006	else
3007	{
3008	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3009	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3010	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3011	}
3012
3013	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3014	bool fInterceptMF = false;
3015	if (!(pCtx->cr0 & X86_CR0_NE))
3016	fInterceptMF = true;
3017
3018	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3019	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3020	{
3021	Assert(PDMVmmDevHeapIsEnabled(pVM));
3022	Assert(pVM->hm.s.vmx.pRealModeTSS);
3023	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3024	fInterceptNM = true;
3025	fInterceptMF = true;
3026	}
3027	else
3028	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3029
3030	if (fInterceptNM)
3031	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3032	else
3033	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3034
3035	if (fInterceptMF)
3036	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3037	else
3038	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3039
3040	/* Additional intercepts for debugging, define these yourself explicitly. */
3041	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3042	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3043	\| RT_BIT(X86_XCPT_BP)
3044	\| RT_BIT(X86_XCPT_DB)
3045	\| RT_BIT(X86_XCPT_DE)
3046	\| RT_BIT(X86_XCPT_NM)
3047	\| RT_BIT(X86_XCPT_UD)
3048	\| RT_BIT(X86_XCPT_NP)
3049	\| RT_BIT(X86_XCPT_SS)
3050	\| RT_BIT(X86_XCPT_GP)
3051	\| RT_BIT(X86_XCPT_PF)
3052	\| RT_BIT(X86_XCPT_MF)
3053	;
3054	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3055	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3056	#endif
3057
3058	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3059
3060	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3061	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3062	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3063	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3064	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3065	else
3066	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3067
3068	u32GuestCR0 \|= uSetCR0;
3069	u32GuestCR0 &= uZapCR0;
3070	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3071
3072	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3073	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3074	AssertRCReturn(rc, rc);
3075	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3076	AssertRCReturn(rc, rc);
3077	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3078
3079	/*
3080	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3081	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3082	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3083	*/
3084	uint32_t u32CR0Mask = 0;
3085	u32CR0Mask = X86_CR0_PE
3086	\| X86_CR0_NE
3087	\| X86_CR0_WP
3088	\| X86_CR0_PG
3089	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3090	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3091	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3092	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3093	u32CR0Mask &= ~X86_CR0_PE;
3094	if (pVM->hm.s.fNestedPaging)
3095	u32CR0Mask &= ~X86_CR0_WP;
3096
3097	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3098	if (fInterceptNM)
3099	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3100	else
3101	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3102
3103	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3104	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3105	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3106	AssertRCReturn(rc, rc);
3107
3108	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3109	}
3110
3111	/*
3112	* Guest CR2.
3113	* It's always loaded in the assembler code. Nothing to do here.
3114	*/
3115
3116	/*
3117	* Guest CR3.
3118	*/
3119	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3120	{
3121	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3122	if (pVM->hm.s.fNestedPaging)
3123	{
3124	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3125
3126	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3127	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3128	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3129	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3130
3131	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3132	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3133	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3134
3135	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3136	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3137	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3138	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3139
3140	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3141	AssertRCReturn(rc, rc);
3142	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3143
3144	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3145	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3146	{
3147	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3148	if (CPUMIsGuestInPAEModeEx(pCtx))
3149	{
3150	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3151	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3152	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3153	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3154	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3155	}
3156
3157	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3158	have Unrestricted Execution to handle the guest when it's not using paging. */
3159	GCPhysGuestCR3 = pCtx->cr3;
3160	}
3161	else
3162	{
3163	/*
3164	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3165	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3166	* EPT takes care of translating it to host-physical addresses.
3167	*/
3168	RTGCPHYS GCPhys;
3169	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3170	Assert(PDMVmmDevHeapIsEnabled(pVM));
3171
3172	/* We obtain it here every time as the guest could have relocated this PCI region. */
3173	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3174	AssertRCReturn(rc, rc);
3175
3176	GCPhysGuestCR3 = GCPhys;
3177	}
3178
3179	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3180	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3181	}
3182	else
3183	{
3184	/* Non-nested paging case, just use the hypervisor's CR3. */
3185	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3186
3187	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3188	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3189	}
3190	AssertRCReturn(rc, rc);
3191
3192	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3193	}
3194
3195	/*
3196	* Guest CR4.
3197	*/
3198	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3199	{
3200	Assert(!(pCtx->cr4 >> 32));
3201	uint32_t u32GuestCR4 = pCtx->cr4;
3202
3203	/* The guest's view of its CR4 is unblemished. */
3204	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3205	AssertRCReturn(rc, rc);
3206	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3207
3208	/* Setup VT-x's view of the guest CR4. */
3209	/*
3210	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3211	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3212	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3213	*/
3214	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3215	{
3216	Assert(pVM->hm.s.vmx.pRealModeTSS);
3217	Assert(PDMVmmDevHeapIsEnabled(pVM));
3218	u32GuestCR4 &= ~X86_CR4_VME;
3219	}
3220
3221	if (pVM->hm.s.fNestedPaging)
3222	{
3223	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3224	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3225	{
3226	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3227	u32GuestCR4 \|= X86_CR4_PSE;
3228	/* Our identity mapping is a 32 bits page directory. */
3229	u32GuestCR4 &= ~X86_CR4_PAE;
3230	}
3231	/* else use guest CR4.*/
3232	}
3233	else
3234	{
3235	/*
3236	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3237	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3238	*/
3239	switch (pVCpu->hm.s.enmShadowMode)
3240	{
3241	case PGMMODE_REAL: /* Real-mode. */
3242	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3243	case PGMMODE_32_BIT: /* 32-bit paging. */
3244	{
3245	u32GuestCR4 &= ~X86_CR4_PAE;
3246	break;
3247	}
3248
3249	case PGMMODE_PAE: /* PAE paging. */
3250	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3251	{
3252	u32GuestCR4 \|= X86_CR4_PAE;
3253	break;
3254	}
3255
3256	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3257	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3258	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3259	break;
3260	#endif
3261	default:
3262	AssertFailed();
3263	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3264	}
3265	}
3266
3267	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3268	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3269	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3270	u32GuestCR4 \|= uSetCR4;
3271	u32GuestCR4 &= uZapCR4;
3272
3273	/* Write VT-x's view of the guest CR4 into the VMCS. */
3274	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3275	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3276	AssertRCReturn(rc, rc);
3277
3278	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3279	uint32_t u32CR4Mask = 0;
3280	u32CR4Mask = X86_CR4_VME
3281	\| X86_CR4_PAE
3282	\| X86_CR4_PGE
3283	\| X86_CR4_PSE
3284	\| X86_CR4_VMXE;
3285	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3286	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3287	AssertRCReturn(rc, rc);
3288
3289	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3290	}
3291	return rc;
3292	}
3293
3294
3295	/**
3296	* Loads the guest debug registers into the guest-state area in the VMCS.
3297	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3298	*
3299	* @returns VBox status code.
3300	* @param pVCpu Pointer to the VMCPU.
3301	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3302	* out-of-sync. Make sure to update the required fields
3303	* before using them.
3304	*
3305	* @remarks No-long-jump zone!!!
3306	*/
3307	static int hmR0VmxLoadGuestDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3308	{
3309	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3310	return VINF_SUCCESS;
3311
3312	#ifdef VBOX_STRICT
3313	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3314	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3315	{
3316	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3317	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3318	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3319	}
3320	#endif
3321
3322	int rc;
3323	PVM pVM = pVCpu->CTX_SUFF(pVM);
3324	bool fInterceptDB = false;
3325	bool fInterceptMovDRx = false;
3326	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3327	{
3328	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3329	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3330	{
3331	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3332	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3333	AssertRCReturn(rc, rc);
3334	Assert(fInterceptDB == false);
3335	}
3336	else
3337	{
3338	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3339	pVCpu->hm.s.fClearTrapFlag = true;
3340	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3341	fInterceptDB = true;
3342	}
3343	}
3344
3345	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3346	{
3347	/*
3348	* Use the combined guest and host DRx values found in the hypervisor
3349	* register set because the debugger has breakpoints active or someone
3350	* is single stepping on the host side without a monitor trap flag.
3351	*
3352	* Note! DBGF expects a clean DR6 state before executing guest code.
3353	*/
3354	if (!CPUMIsHyperDebugStateActive(pVCpu))
3355	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3356	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3357	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3358
3359	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3360	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3361	AssertRCReturn(rc, rc);
3362
3363	fInterceptDB = true;
3364	fInterceptMovDRx = true;
3365	}
3366	else
3367	{
3368	/*
3369	* If the guest has enabled debug registers, we need to load them prior to
3370	* executing guest code so they'll trigger at the right time.
3371	*/
3372	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3373	{
3374	if (!CPUMIsGuestDebugStateActive(pVCpu))
3375	{
3376	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3377	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3378	}
3379	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3380	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3381	}
3382	/*
3383	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3384	* must intercept #DB in order to maintain a correct DR6 guest value.
3385	*/
3386	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3387	{
3388	fInterceptMovDRx = true;
3389	fInterceptDB = true;
3390	}
3391
3392	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3393	AssertRCReturn(rc, rc);
3394	}
3395
3396	/*
3397	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3398	*/
3399	if (fInterceptDB)
3400	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3401	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3402	{
3403	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3404	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3405	#endif
3406	}
3407	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3408	AssertRCReturn(rc, rc);
3409
3410	/*
3411	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3412	*/
3413	if (fInterceptMovDRx)
3414	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3415	else
3416	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3417	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3418	AssertRCReturn(rc, rc);
3419
3420	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3421	return VINF_SUCCESS;
3422	}
3423
3424
3425	#ifdef VBOX_STRICT
3426	/**
3427	* Strict function to validate segment registers.
3428	*
3429	* @remarks Requires CR0.
3430	*/
3431	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3432	{
3433	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3434	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3435	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3436	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3437	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3438	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3439	{
3440	/* Protected mode checks */
3441	/* CS */
3442	Assert(pCtx->cs.Attr.n.u1Present);
3443	Assert(!(pCtx->cs.Attr.u & 0xf00));
3444	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3445	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3446	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3447	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3448	\|\| (pCtx->cs.Attr.n.u1Granularity));
3449	/* CS cannot be loaded with NULL in protected mode. */
3450	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3451	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3452	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3453	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3454	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3455	else
3456	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3457	/* SS */
3458	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3459	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3460	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3461	if ( !(pCtx->cr0 & X86_CR0_PE)
3462	\|\| pCtx->cs.Attr.n.u4Type == 3)
3463	{
3464	Assert(!pCtx->ss.Attr.n.u2Dpl);
3465	}
3466	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3467	{
3468	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3469	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3470	Assert(pCtx->ss.Attr.n.u1Present);
3471	Assert(!(pCtx->ss.Attr.u & 0xf00));
3472	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3473	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3474	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3475	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3476	\|\| (pCtx->ss.Attr.n.u1Granularity));
3477	}
3478	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3479	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3480	{
3481	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3482	Assert(pCtx->ds.Attr.n.u1Present);
3483	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3484	Assert(!(pCtx->ds.Attr.u & 0xf00));
3485	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3486	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3487	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3488	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3489	\|\| (pCtx->ds.Attr.n.u1Granularity));
3490	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3491	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3492	}
3493	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3494	{
3495	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3496	Assert(pCtx->es.Attr.n.u1Present);
3497	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3498	Assert(!(pCtx->es.Attr.u & 0xf00));
3499	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3500	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3501	\|\| !(pCtx->es.Attr.n.u1Granularity));
3502	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3503	\|\| (pCtx->es.Attr.n.u1Granularity));
3504	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3505	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3506	}
3507	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3508	{
3509	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3510	Assert(pCtx->fs.Attr.n.u1Present);
3511	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3512	Assert(!(pCtx->fs.Attr.u & 0xf00));
3513	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3514	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3515	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3516	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3517	\|\| (pCtx->fs.Attr.n.u1Granularity));
3518	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3519	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3520	}
3521	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3522	{
3523	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3524	Assert(pCtx->gs.Attr.n.u1Present);
3525	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3526	Assert(!(pCtx->gs.Attr.u & 0xf00));
3527	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3528	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3529	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3530	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3531	\|\| (pCtx->gs.Attr.n.u1Granularity));
3532	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3533	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3534	}
3535	/* 64-bit capable CPUs. */
3536	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3537	Assert(!(pCtx->cs.u64Base >> 32));
3538	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3539	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3540	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3541	# endif
3542	}
3543	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3544	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3545	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3546	{
3547	/* Real and v86 mode checks. */
3548	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3549	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3550	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3551	{
3552	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3553	}
3554	else
3555	{
3556	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3557	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3558	}
3559
3560	/* CS */
3561	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3562	Assert(pCtx->cs.u32Limit == 0xffff);
3563	Assert(u32CSAttr == 0xf3);
3564	/* SS */
3565	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3566	Assert(pCtx->ss.u32Limit == 0xffff);
3567	Assert(u32SSAttr == 0xf3);
3568	/* DS */
3569	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3570	Assert(pCtx->ds.u32Limit == 0xffff);
3571	Assert(u32DSAttr == 0xf3);
3572	/* ES */
3573	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3574	Assert(pCtx->es.u32Limit == 0xffff);
3575	Assert(u32ESAttr == 0xf3);
3576	/* FS */
3577	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3578	Assert(pCtx->fs.u32Limit == 0xffff);
3579	Assert(u32FSAttr == 0xf3);
3580	/* GS */
3581	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3582	Assert(pCtx->gs.u32Limit == 0xffff);
3583	Assert(u32GSAttr == 0xf3);
3584	/* 64-bit capable CPUs. */
3585	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3586	Assert(!(pCtx->cs.u64Base >> 32));
3587	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3588	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3589	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3590	# endif
3591	}
3592	}
3593	#endif /* VBOX_STRICT */
3594
3595
3596	/**
3597	* Writes a guest segment register into the guest-state area in the VMCS.
3598	*
3599	* @returns VBox status code.
3600	* @param pVCpu Pointer to the VMCPU.
3601	* @param idxSel Index of the selector in the VMCS.
3602	* @param idxLimit Index of the segment limit in the VMCS.
3603	* @param idxBase Index of the segment base in the VMCS.
3604	* @param idxAccess Index of the access rights of the segment in the VMCS.
3605	* @param pSelReg Pointer to the segment selector.
3606	* @param pCtx Pointer to the guest-CPU context.
3607	*
3608	* @remarks No-long-jump zone!!!
3609	*/
3610	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3611	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3612	{
3613	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3614	AssertRCReturn(rc, rc);
3615	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3616	AssertRCReturn(rc, rc);
3617	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3618	AssertRCReturn(rc, rc);
3619
3620	uint32_t u32Access = pSelReg->Attr.u;
3621	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3622	{
3623	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3624	u32Access = 0xf3;
3625	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3626	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3627	}
3628	else
3629	{
3630	/*
3631	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3632	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3633	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3634	* loaded in protected-mode have their attribute as 0.
3635	*/
3636	if (!u32Access)
3637	u32Access = X86DESCATTR_UNUSABLE;
3638	}
3639
3640	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3641	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3642	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3643
3644	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3645	AssertRCReturn(rc, rc);
3646	return rc;
3647	}
3648
3649
3650	/**
3651	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3652	* into the guest-state area in the VMCS.
3653	*
3654	* @returns VBox status code.
3655	* @param pVM Pointer to the VM.
3656	* @param pVCPU Pointer to the VMCPU.
3657	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3658	* out-of-sync. Make sure to update the required fields
3659	* before using them.
3660	*
3661	* @remarks Requires CR0 (strict builds validation).
3662	* @remarks No-long-jump zone!!!
3663	*/
3664	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3665	{
3666	int rc = VERR_INTERNAL_ERROR_5;
3667	PVM pVM = pVCpu->CTX_SUFF(pVM);
3668
3669	/*
3670	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3671	*/
3672	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3673	{
3674	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3675	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3676	{
3677	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3678	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3679	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3680	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3681	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3682	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3683	}
3684
3685	#ifdef VBOX_WITH_REM
3686	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3687	{
3688	Assert(pVM->hm.s.vmx.pRealModeTSS);
3689	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3690	if ( pVCpu->hm.s.vmx.fWasInRealMode
3691	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3692	{
3693	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3694	in real-mode (e.g. OpenBSD 4.0) */
3695	REMFlushTBs(pVM);
3696	Log4(("Load: Switch to protected mode detected!\n"));
3697	pVCpu->hm.s.vmx.fWasInRealMode = false;
3698	}
3699	}
3700	#endif
3701	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3702	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3703	AssertRCReturn(rc, rc);
3704	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3705	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3706	AssertRCReturn(rc, rc);
3707	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3708	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3709	AssertRCReturn(rc, rc);
3710	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3711	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3712	AssertRCReturn(rc, rc);
3713	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3714	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3715	AssertRCReturn(rc, rc);
3716	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3717	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3718	AssertRCReturn(rc, rc);
3719
3720	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3721	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3722	#ifdef VBOX_STRICT
3723	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3724	#endif
3725	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3726	}
3727
3728	/*
3729	* Guest TR.
3730	*/
3731	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3732	{
3733	/*
3734	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3735	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3736	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3737	*/
3738	uint16_t u16Sel = 0;
3739	uint32_t u32Limit = 0;
3740	uint64_t u64Base = 0;
3741	uint32_t u32AccessRights = 0;
3742
3743	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3744	{
3745	u16Sel = pMixedCtx->tr.Sel;
3746	u32Limit = pMixedCtx->tr.u32Limit;
3747	u64Base = pMixedCtx->tr.u64Base;
3748	u32AccessRights = pMixedCtx->tr.Attr.u;
3749	}
3750	else
3751	{
3752	Assert(pVM->hm.s.vmx.pRealModeTSS);
3753	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3754
3755	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3756	RTGCPHYS GCPhys;
3757	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3758	AssertRCReturn(rc, rc);
3759
3760	X86DESCATTR DescAttr;
3761	DescAttr.u = 0;
3762	DescAttr.n.u1Present = 1;
3763	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3764
3765	u16Sel = 0;
3766	u32Limit = HM_VTX_TSS_SIZE;
3767	u64Base = GCPhys; /* in real-mode phys = virt. */
3768	u32AccessRights = DescAttr.u;
3769	}
3770
3771	/* Validate. */
3772	Assert(!(u16Sel & RT_BIT(2)));
3773	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3774	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3775	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3776	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3777	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3778	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3779	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3780	Assert( (u32Limit & 0xfff) == 0xfff
3781	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3782	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3783	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3784
3785	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3786	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3787	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3788	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3789
3790	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3791	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3792	}
3793
3794	/*
3795	* Guest GDTR.
3796	*/
3797	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3798	{
3799	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3800	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3801
3802	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3803	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3804	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3805	}
3806
3807	/*
3808	* Guest LDTR.
3809	*/
3810	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3811	{
3812	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3813	uint32_t u32Access = 0;
3814	if (!pMixedCtx->ldtr.Attr.u)
3815	u32Access = X86DESCATTR_UNUSABLE;
3816	else
3817	u32Access = pMixedCtx->ldtr.Attr.u;
3818
3819	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3820	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3821	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3822	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3823
3824	/* Validate. */
3825	if (!(u32Access & X86DESCATTR_UNUSABLE))
3826	{
3827	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3828	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3829	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3830	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3831	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3832	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3833	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3834	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3835	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3836	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3837	}
3838
3839	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3840	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3841	}
3842
3843	/*
3844	* Guest IDTR.
3845	*/
3846	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3847	{
3848	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3849	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3850
3851	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3852	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3853	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3854	}
3855
3856	return VINF_SUCCESS;
3857	}
3858
3859
3860	/**
3861	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3862	* areas. These MSRs will automatically be loaded to the host CPU on every
3863	* successful VM entry and stored from the host CPU on every successful VM exit.
3864	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3865	*
3866	* @returns VBox status code.
3867	* @param pVCpu Pointer to the VMCPU.
3868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3869	* out-of-sync. Make sure to update the required fields
3870	* before using them.
3871	*
3872	* @remarks No-long-jump zone!!!
3873	*/
3874	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3875	{
3876	AssertPtr(pVCpu);
3877	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3878
3879	/*
3880	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3881	*/
3882	int rc = VINF_SUCCESS;
3883	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3884	{
3885	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3886	PVM pVM = pVCpu->CTX_SUFF(pVM);
3887	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3888	uint32_t cGuestMsrs = 0;
3889
3890	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3891	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3892	* when the guest really is in 64-bit mode. */
3893	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3894	if (fSupportsLongMode)
3895	{
3896	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3897	pGuestMsr->u32Reserved = 0;
3898	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3899	pGuestMsr++; cGuestMsrs++;
3900	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3901	pGuestMsr->u32Reserved = 0;
3902	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3903	pGuestMsr++; cGuestMsrs++;
3904	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3905	pGuestMsr->u32Reserved = 0;
3906	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3907	pGuestMsr++; cGuestMsrs++;
3908	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3909	pGuestMsr->u32Reserved = 0;
3910	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3911	pGuestMsr++; cGuestMsrs++;
3912	}
3913
3914	/*
3915	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3916	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3917	*/
3918	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3919	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3920	{
3921	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3922	pGuestMsr->u32Reserved = 0;
3923	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3924	AssertRCReturn(rc, rc);
3925	pGuestMsr++; cGuestMsrs++;
3926	}
3927
3928	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3929	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3930	{
3931	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3932	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3933	}
3934
3935	/* Update the VCPU's copy of the guest MSR count. */
3936	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3937	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3938	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3939	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3940
3941	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3942	}
3943
3944	/*
3945	* Guest Sysenter MSRs.
3946	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3947	* VM-exits on WRMSRs for these MSRs.
3948	*/
3949	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3950	{
3951	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3952	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3953	}
3954	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3955	{
3956	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3957	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3958	}
3959	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3960	{
3961	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3962	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3963	}
3964
3965	return rc;
3966	}
3967
3968
3969	/**
3970	* Loads the guest activity state into the guest-state area in the VMCS.
3971	*
3972	* @returns VBox status code.
3973	* @param pVCpu Pointer to the VMCPU.
3974	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3975	* out-of-sync. Make sure to update the required fields
3976	* before using them.
3977	*
3978	* @remarks No-long-jump zone!!!
3979	*/
3980	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3981	{
3982	/** @todo See if we can make use of other states, e.g.
3983	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3984	int rc = VINF_SUCCESS;
3985	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3986	{
3987	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3988	AssertRCReturn(rc, rc);
3989	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3990	}
3991	return rc;
3992	}
3993
3994
3995	/**
3996	* Sets up the appropriate function to run guest code.
3997	*
3998	* @returns VBox status code.
3999	* @param pVCpu Pointer to the VMCPU.
4000	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4001	* out-of-sync. Make sure to update the required fields
4002	* before using them.
4003	*
4004	* @remarks No-long-jump zone!!!
4005	*/
4006	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4007	{
4008	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4009	{
4010	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4011	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4012	#endif
4013	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4014	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4015	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4016	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4017	#else
4018	/* 64-bit host or hybrid host. */
4019	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4020	#endif
4021	}
4022	else
4023	{
4024	/* Guest is not in long mode, use the 32-bit handler. */
4025	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4026	}
4027	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4028	return VINF_SUCCESS;
4029	}
4030
4031
4032	/**
4033	* Wrapper for running the guest code in VT-x.
4034	*
4035	* @returns VBox strict status code.
4036	* @param pVM Pointer to the VM.
4037	* @param pVCpu Pointer to the VMCPU.
4038	* @param pCtx Pointer to the guest-CPU context.
4039	*
4040	* @remarks No-long-jump zone!!!
4041	*/
4042	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4043	{
4044	/*
4045	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4046	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4047	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4048	*/
4049	#ifdef VBOX_WITH_KERNEL_USING_XMM
4050	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4051	#else
4052	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4053	#endif
4054	}
4055
4056
4057	/**
4058	* Reports world-switch error and dumps some useful debug info.
4059	*
4060	* @param pVM Pointer to the VM.
4061	* @param pVCpu Pointer to the VMCPU.
4062	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4063	* @param pCtx Pointer to the guest-CPU context.
4064	* @param pVmxTransient Pointer to the VMX transient structure (only
4065	* exitReason updated).
4066	*/
4067	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4068	{
4069	Assert(pVM);
4070	Assert(pVCpu);
4071	Assert(pCtx);
4072	Assert(pVmxTransient);
4073	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4074
4075	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4076	switch (rcVMRun)
4077	{
4078	case VERR_VMX_INVALID_VMXON_PTR:
4079	AssertFailed();
4080	break;
4081	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4082	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4083	{
4084	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4085	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4086	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4087	AssertRC(rc);
4088
4089	#ifdef VBOX_STRICT
4090	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4091	pVmxTransient->uExitReason));
4092	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4093	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4094	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4095	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4096	else
4097	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4098
4099	/* VMX control bits. */
4100	uint32_t u32Val;
4101	uint64_t u64Val;
4102	HMVMXHCUINTREG uHCReg;
4103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4104	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4106	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4108	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4110	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4112	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4114	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4116	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4118	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4120	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4121	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4122	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4123	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4124	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4125	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4126	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4128	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4129	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4130	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4131	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4132	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4133	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4134	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4135	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4136	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4137	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4138	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4139	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4140	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4141	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4142	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4143	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4144	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4145
4146	/* Guest bits. */
4147	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4148	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4149	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4150	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4151	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4152	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4153	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4154	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4155
4156	/* Host bits. */
4157	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4158	Log4(("Host CR0 %#RHr\n", uHCReg));
4159	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4160	Log4(("Host CR3 %#RHr\n", uHCReg));
4161	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4162	Log4(("Host CR4 %#RHr\n", uHCReg));
4163
4164	RTGDTR HostGdtr;
4165	PCX86DESCHC pDesc;
4166	ASMGetGDTR(&HostGdtr);
4167	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4168	Log4(("Host CS %#08x\n", u32Val));
4169	if (u32Val < HostGdtr.cbGdt)
4170	{
4171	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4172	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4173	}
4174
4175	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4176	Log4(("Host DS %#08x\n", u32Val));
4177	if (u32Val < HostGdtr.cbGdt)
4178	{
4179	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4180	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4181	}
4182
4183	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4184	Log4(("Host ES %#08x\n", u32Val));
4185	if (u32Val < HostGdtr.cbGdt)
4186	{
4187	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4188	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4189	}
4190
4191	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4192	Log4(("Host FS %#08x\n", u32Val));
4193	if (u32Val < HostGdtr.cbGdt)
4194	{
4195	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4196	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4197	}
4198
4199	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4200	Log4(("Host GS %#08x\n", u32Val));
4201	if (u32Val < HostGdtr.cbGdt)
4202	{
4203	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4204	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4205	}
4206
4207	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4208	Log4(("Host SS %#08x\n", u32Val));
4209	if (u32Val < HostGdtr.cbGdt)
4210	{
4211	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4212	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4213	}
4214
4215	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4216	Log4(("Host TR %#08x\n", u32Val));
4217	if (u32Val < HostGdtr.cbGdt)
4218	{
4219	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4220	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4221	}
4222
4223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4224	Log4(("Host TR Base %#RHv\n", uHCReg));
4225	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4226	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4227	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4228	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4229	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4230	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4231	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4232	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4233	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4234	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4235	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4236	Log4(("Host RSP %#RHv\n", uHCReg));
4237	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4238	Log4(("Host RIP %#RHv\n", uHCReg));
4239	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4240	if (HMVMX_IS_64BIT_HOST_MODE())
4241	{
4242	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4243	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4244	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4245	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4246	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4247	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4248	}
4249	# endif
4250	#endif /* VBOX_STRICT */
4251	break;
4252	}
4253
4254	default:
4255	/* Impossible */
4256	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4257	break;
4258	}
4259	NOREF(pVM);
4260	}
4261
4262
4263	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4264	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4265	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4266	#endif
4267	#ifdef VBOX_STRICT
4268	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4269	{
4270	switch (idxField)
4271	{
4272	case VMX_VMCS_GUEST_RIP:
4273	case VMX_VMCS_GUEST_RSP:
4274	case VMX_VMCS_GUEST_SYSENTER_EIP:
4275	case VMX_VMCS_GUEST_SYSENTER_ESP:
4276	case VMX_VMCS_GUEST_GDTR_BASE:
4277	case VMX_VMCS_GUEST_IDTR_BASE:
4278	case VMX_VMCS_GUEST_CS_BASE:
4279	case VMX_VMCS_GUEST_DS_BASE:
4280	case VMX_VMCS_GUEST_ES_BASE:
4281	case VMX_VMCS_GUEST_FS_BASE:
4282	case VMX_VMCS_GUEST_GS_BASE:
4283	case VMX_VMCS_GUEST_SS_BASE:
4284	case VMX_VMCS_GUEST_LDTR_BASE:
4285	case VMX_VMCS_GUEST_TR_BASE:
4286	case VMX_VMCS_GUEST_CR3:
4287	return true;
4288	}
4289	return false;
4290	}
4291
4292	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4293	{
4294	switch (idxField)
4295	{
4296	/* Read-only fields. */
4297	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4298	return true;
4299	}
4300	/* Remaining readable fields should also be writable. */
4301	return hmR0VmxIsValidWriteField(idxField);
4302	}
4303	#endif /* VBOX_STRICT */
4304
4305
4306	/**
4307	* Executes the specified handler in 64-bit mode.
4308	*
4309	* @returns VBox status code.
4310	* @param pVM Pointer to the VM.
4311	* @param pVCpu Pointer to the VMCPU.
4312	* @param pCtx Pointer to the guest CPU context.
4313	* @param enmOp The operation to perform.
4314	* @param cbParam Number of parameters.
4315	* @param paParam Array of 32-bit parameters.
4316	*/
4317	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4318	uint32_t *paParam)
4319	{
4320	int rc, rc2;
4321	PHMGLOBLCPUINFO pCpu;
4322	RTHCPHYS HCPhysCpuPage;
4323	RTCCUINTREG uOldEFlags;
4324
4325	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4326	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4327	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4328	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4329
4330	#ifdef VBOX_STRICT
4331	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4332	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4333
4334	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4335	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4336	#endif
4337
4338	/* Disable interrupts. */
4339	uOldEFlags = ASMIntDisableFlags();
4340
4341	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4342	RTCPUID idHostCpu = RTMpCpuId();
4343	CPUMR0SetLApic(pVM, idHostCpu);
4344	#endif
4345
4346	pCpu = HMR0GetCurrentCpu();
4347	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4348
4349	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4350	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4351
4352	/* Leave VMX Root Mode. */
4353	VMXDisable();
4354
4355	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4356
4357	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4358	CPUMSetHyperEIP(pVCpu, enmOp);
4359	for (int i = (int)cbParam - 1; i >= 0; i--)
4360	CPUMPushHyper(pVCpu, paParam[i]);
4361
4362	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4363
4364	/* Call the switcher. */
4365	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4366	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4367
4368	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4369	/* Make sure the VMX instructions don't cause #UD faults. */
4370	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4371
4372	/* Re-enter VMX Root Mode */
4373	rc2 = VMXEnable(HCPhysCpuPage);
4374	if (RT_FAILURE(rc2))
4375	{
4376	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4377	ASMSetFlags(uOldEFlags);
4378	return rc2;
4379	}
4380
4381	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4382	AssertRC(rc2);
4383	Assert(!(ASMGetFlags() & X86_EFL_IF));
4384	ASMSetFlags(uOldEFlags);
4385	return rc;
4386	}
4387
4388
4389	/**
4390	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4391	* supporting 64-bit guests.
4392	*
4393	* @returns VBox status code.
4394	* @param fResume Whether to VMLAUNCH or VMRESUME.
4395	* @param pCtx Pointer to the guest-CPU context.
4396	* @param pCache Pointer to the VMCS cache.
4397	* @param pVM Pointer to the VM.
4398	* @param pVCpu Pointer to the VMCPU.
4399	*/
4400	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4401	{
4402	uint32_t aParam[6];
4403	PHMGLOBLCPUINFO pCpu = NULL;
4404	RTHCPHYS HCPhysCpuPage = 0;
4405	int rc = VERR_INTERNAL_ERROR_5;
4406
4407	pCpu = HMR0GetCurrentCpu();
4408	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4409
4410	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4411	pCache->uPos = 1;
4412	pCache->interPD = PGMGetInterPaeCR3(pVM);
4413	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4414	#endif
4415
4416	#ifdef VBOX_STRICT
4417	pCache->TestIn.HCPhysCpuPage = 0;
4418	pCache->TestIn.HCPhysVmcs = 0;
4419	pCache->TestIn.pCache = 0;
4420	pCache->TestOut.HCPhysVmcs = 0;
4421	pCache->TestOut.pCache = 0;
4422	pCache->TestOut.pCtx = 0;
4423	pCache->TestOut.eflags = 0;
4424	#endif
4425
4426	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4427	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4428	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4429	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4430	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4431	aParam[5] = 0;
4432
4433	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4434	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4435	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4436	#endif
4437	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4438
4439	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4440	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4441	Assert(pCtx->dr[4] == 10);
4442	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4443	#endif
4444
4445	#ifdef VBOX_STRICT
4446	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4447	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4448	pVCpu->hm.s.vmx.HCPhysVmcs));
4449	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4450	pCache->TestOut.HCPhysVmcs));
4451	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4452	pCache->TestOut.pCache));
4453	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4454	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4455	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4456	pCache->TestOut.pCtx));
4457	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4458	#endif
4459	return rc;
4460	}
4461
4462
4463	/**
4464	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4465	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4466	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4467	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4468	*
4469	* @returns VBox status code.
4470	* @param pVM Pointer to the VM.
4471	* @param pVCpu Pointer to the VMCPU.
4472	*/
4473	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4474	{
4475	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4476	{ \
4477	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4478	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4479	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4480	++cReadFields; \
4481	}
4482
4483	AssertPtr(pVM);
4484	AssertPtr(pVCpu);
4485	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4486	uint32_t cReadFields = 0;
4487
4488	/*
4489	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4490	* and serve to indicate exceptions to the rules.
4491	*/
4492
4493	/* Guest-natural selector base fields. */
4494	#if 0
4495	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4498	#endif
4499	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4500	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4501	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4503	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4511	#if 0
4512	/* Unused natural width guest-state fields. */
4513	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4515	#endif
4516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4518
4519	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4520	#if 0
4521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4523	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4530	#endif
4531
4532	/* Natural width guest-state fields. */
4533	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4534	#if 0
4535	/* Currently unused field. */
4536	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4537	#endif
4538
4539	if (pVM->hm.s.fNestedPaging)
4540	{
4541	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4542	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4543	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4544	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4545	}
4546	else
4547	{
4548	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4549	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4550	}
4551
4552	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4553	return VINF_SUCCESS;
4554	}
4555
4556
4557	/**
4558	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4559	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4560	* darwin, running 64-bit guests).
4561	*
4562	* @returns VBox status code.
4563	* @param pVCpu Pointer to the VMCPU.
4564	* @param idxField The VMCS field encoding.
4565	* @param u64Val 16, 32 or 64 bits value.
4566	*/
4567	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4568	{
4569	int rc;
4570	switch (idxField)
4571	{
4572	/*
4573	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4574	*/
4575	/* 64-bit Control fields. */
4576	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4577	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4578	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4579	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4580	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4581	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4582	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4583	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4584	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4585	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4586	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4587	case VMX_VMCS64_CTRL_EPTP_FULL:
4588	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4589	/* 64-bit Guest-state fields. */
4590	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4591	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4592	case VMX_VMCS64_GUEST_PAT_FULL:
4593	case VMX_VMCS64_GUEST_EFER_FULL:
4594	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4595	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4596	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4597	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4598	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4599	/* 64-bit Host-state fields. */
4600	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4601	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4602	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4603	{
4604	rc = VMXWriteVmcs32(idxField, u64Val);
4605	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4606	break;
4607	}
4608
4609	/*
4610	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4611	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4612	*/
4613	/* Natural-width Guest-state fields. */
4614	case VMX_VMCS_GUEST_CR3:
4615	case VMX_VMCS_GUEST_ES_BASE:
4616	case VMX_VMCS_GUEST_CS_BASE:
4617	case VMX_VMCS_GUEST_SS_BASE:
4618	case VMX_VMCS_GUEST_DS_BASE:
4619	case VMX_VMCS_GUEST_FS_BASE:
4620	case VMX_VMCS_GUEST_GS_BASE:
4621	case VMX_VMCS_GUEST_LDTR_BASE:
4622	case VMX_VMCS_GUEST_TR_BASE:
4623	case VMX_VMCS_GUEST_GDTR_BASE:
4624	case VMX_VMCS_GUEST_IDTR_BASE:
4625	case VMX_VMCS_GUEST_RSP:
4626	case VMX_VMCS_GUEST_RIP:
4627	case VMX_VMCS_GUEST_SYSENTER_ESP:
4628	case VMX_VMCS_GUEST_SYSENTER_EIP:
4629	{
4630	if (!(u64Val >> 32))
4631	{
4632	/* If this field is 64-bit, VT-x will zero out the top bits. */
4633	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4634	}
4635	else
4636	{
4637	/* Assert that only the 32->64 switcher case should ever come here. */
4638	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4639	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4640	}
4641	break;
4642	}
4643
4644	default:
4645	{
4646	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4647	rc = VERR_INVALID_PARAMETER;
4648	break;
4649	}
4650	}
4651	AssertRCReturn(rc, rc);
4652	return rc;
4653	}
4654
4655
4656	/**
4657	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4658	* hosts (except darwin) for 64-bit guests.
4659	*
4660	* @param pVCpu Pointer to the VMCPU.
4661	* @param idxField The VMCS field encoding.
4662	* @param u64Val 16, 32 or 64 bits value.
4663	*/
4664	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4665	{
4666	AssertPtr(pVCpu);
4667	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4668
4669	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4670	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4671
4672	/* Make sure there are no duplicates. */
4673	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4674	{
4675	if (pCache->Write.aField[i] == idxField)
4676	{
4677	pCache->Write.aFieldVal[i] = u64Val;
4678	return VINF_SUCCESS;
4679	}
4680	}
4681
4682	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4683	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4684	pCache->Write.cValidEntries++;
4685	return VINF_SUCCESS;
4686	}
4687
4688	/* Enable later when the assembly code uses these as callbacks. */
4689	#if 0
4690	/*
4691	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4692	*
4693	* @param pVCpu Pointer to the VMCPU.
4694	* @param pCache Pointer to the VMCS cache.
4695	*
4696	* @remarks No-long-jump zone!!!
4697	*/
4698	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4699	{
4700	AssertPtr(pCache);
4701	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4702	{
4703	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4704	AssertRC(rc);
4705	}
4706	pCache->Write.cValidEntries = 0;
4707	}
4708
4709
4710	/**
4711	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4712	*
4713	* @param pVCpu Pointer to the VMCPU.
4714	* @param pCache Pointer to the VMCS cache.
4715	*
4716	* @remarks No-long-jump zone!!!
4717	*/
4718	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4719	{
4720	AssertPtr(pCache);
4721	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4722	{
4723	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4724	AssertRC(rc);
4725	}
4726	}
4727	#endif
4728	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4729
4730
4731	/**
4732	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4733	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4734	* timer.
4735	*
4736	* @returns VBox status code.
4737	* @param pVCpu Pointer to the VMCPU.
4738	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4739	* out-of-sync. Make sure to update the required fields
4740	* before using them.
4741	* @remarks No-long-jump zone!!!
4742	*/
4743	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4744	{
4745	int rc = VERR_INTERNAL_ERROR_5;
4746	bool fOffsettedTsc = false;
4747	PVM pVM = pVCpu->CTX_SUFF(pVM);
4748	if (pVM->hm.s.vmx.fUsePreemptTimer)
4749	{
4750	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4751
4752	/* Make sure the returned values have sane upper and lower boundaries. */
4753	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4754	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4755	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4756	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4757
4758	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4759	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4760	}
4761	else
4762	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4763
4764	if (fOffsettedTsc)
4765	{
4766	uint64_t u64CurTSC = ASMReadTSC();
4767	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4768	{
4769	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4770	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4771
4772	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4773	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4774	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4775	}
4776	else
4777	{
4778	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4779	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4780	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4781	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4782	}
4783	}
4784	else
4785	{
4786	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4787	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4788	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4789	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4790	}
4791	}
4792
4793
4794	/**
4795	* Determines if an exception is a contributory exception. Contributory
4796	* exceptions are ones which can cause double-faults. Page-fault is
4797	* intentionally not included here as it's a conditional contributory exception.
4798	*
4799	* @returns true if the exception is contributory, false otherwise.
4800	* @param uVector The exception vector.
4801	*/
4802	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4803	{
4804	switch (uVector)
4805	{
4806	case X86_XCPT_GP:
4807	case X86_XCPT_SS:
4808	case X86_XCPT_NP:
4809	case X86_XCPT_TS:
4810	case X86_XCPT_DE:
4811	return true;
4812	default:
4813	break;
4814	}
4815	return false;
4816	}
4817
4818
4819	/**
4820	* Sets an event as a pending event to be injected into the guest.
4821	*
4822	* @param pVCpu Pointer to the VMCPU.
4823	* @param u32IntrInfo The VM-entry interruption-information field.
4824	* @param cbInstr The VM-entry instruction length in bytes (for software
4825	* interrupts, exceptions and privileged software
4826	* exceptions).
4827	* @param u32ErrCode The VM-entry exception error code.
4828	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4829	* page-fault.
4830	*
4831	* @remarks Statistics counter assumes this is a guest event being injected or
4832	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4833	* always incremented.
4834	*/
4835	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4836	RTGCUINTPTR GCPtrFaultAddress)
4837	{
4838	Assert(!pVCpu->hm.s.Event.fPending);
4839	pVCpu->hm.s.Event.fPending = true;
4840	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4841	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4842	pVCpu->hm.s.Event.cbInstr = cbInstr;
4843	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4844
4845	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4846	}
4847
4848
4849	/**
4850	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4851	*
4852	* @param pVCpu Pointer to the VMCPU.
4853	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4854	* out-of-sync. Make sure to update the required fields
4855	* before using them.
4856	*/
4857	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4858	{
4859	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4860	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4861	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4862	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4863	}
4864
4865
4866	/**
4867	* Handle a condition that occurred while delivering an event through the guest
4868	* IDT.
4869	*
4870	* @returns VBox status code (informational error codes included).
4871	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4872	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4873	* continue execution of the guest which will delivery the #DF.
4874	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4875	*
4876	* @param pVCpu Pointer to the VMCPU.
4877	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4878	* out-of-sync. Make sure to update the required fields
4879	* before using them.
4880	* @param pVmxTransient Pointer to the VMX transient structure.
4881	*
4882	* @remarks No-long-jump zone!!!
4883	*/
4884	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4885	{
4886	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4887	AssertRC(rc);
4888	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4889	{
4890	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4891	AssertRCReturn(rc, rc);
4892
4893	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4894	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4895	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4896
4897	typedef enum
4898	{
4899	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4900	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4901	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4902	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4903	} VMXREFLECTXCPT;
4904
4905	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4906	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4907	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4908	{
4909	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4910	{
4911	enmReflect = VMXREFLECTXCPT_XCPT;
4912	#ifdef VBOX_STRICT
4913	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4914	&& uExitVector == X86_XCPT_PF)
4915	{
4916	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4917	}
4918	#endif
4919	if ( uExitVector == X86_XCPT_PF
4920	&& uIdtVector == X86_XCPT_PF)
4921	{
4922	pVmxTransient->fVectoringPF = true;
4923	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4924	}
4925	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4926	&& hmR0VmxIsContributoryXcpt(uExitVector)
4927	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4928	\|\| uIdtVector == X86_XCPT_PF))
4929	{
4930	enmReflect = VMXREFLECTXCPT_DF;
4931	}
4932	else if (uIdtVector == X86_XCPT_DF)
4933	enmReflect = VMXREFLECTXCPT_TF;
4934	}
4935	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4936	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4937	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4938	{
4939	/*
4940	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4941	* (whatever they are) as they reoccur when restarting the instruction.
4942	*/
4943	enmReflect = VMXREFLECTXCPT_XCPT;
4944	}
4945	}
4946	else
4947	{
4948	/*
4949	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4950	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4951	* original exception to the guest after handling the VM-exit.
4952	*/
4953	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4954	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4955	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4956	{
4957	enmReflect = VMXREFLECTXCPT_XCPT;
4958	}
4959	}
4960
4961	switch (enmReflect)
4962	{
4963	case VMXREFLECTXCPT_XCPT:
4964	{
4965	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4966	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4967	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
4968
4969	uint32_t u32ErrCode = 0;
4970	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
4971	{
4972	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4973	AssertRCReturn(rc, rc);
4974	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4975	}
4976
4977	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4978	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4979	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4980	rc = VINF_SUCCESS;
4981	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
4982	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4983
4984	break;
4985	}
4986
4987	case VMXREFLECTXCPT_DF:
4988	{
4989	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4990	rc = VINF_HM_DOUBLE_FAULT;
4991	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
4992	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4993
4994	break;
4995	}
4996
4997	case VMXREFLECTXCPT_TF:
4998	{
4999	rc = VINF_EM_RESET;
5000	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5001	uExitVector));
5002	break;
5003	}
5004
5005	default:
5006	Assert(rc == VINF_SUCCESS);
5007	break;
5008	}
5009	}
5010	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5011	return rc;
5012	}
5013
5014
5015	/**
5016	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5017	*
5018	* @returns VBox status code.
5019	* @param pVCpu Pointer to the VMCPU.
5020	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5021	* out-of-sync. Make sure to update the required fields
5022	* before using them.
5023	*
5024	* @remarks No-long-jump zone!!!
5025	*/
5026	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5027	{
5028	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5029	{
5030	uint32_t uVal = 0;
5031	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5032	AssertRCReturn(rc, rc);
5033	uint32_t uShadow = 0;
5034	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5035	AssertRCReturn(rc, rc);
5036
5037	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5038	CPUMSetGuestCR0(pVCpu, uVal);
5039	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5040	}
5041	return VINF_SUCCESS;
5042	}
5043
5044
5045	/**
5046	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5047	*
5048	* @returns VBox status code.
5049	* @param pVCpu Pointer to the VMCPU.
5050	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5051	* out-of-sync. Make sure to update the required fields
5052	* before using them.
5053	*
5054	* @remarks No-long-jump zone!!!
5055	*/
5056	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5057	{
5058	int rc = VINF_SUCCESS;
5059	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5060	{
5061	uint32_t uVal = 0;
5062	uint32_t uShadow = 0;
5063	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5064	AssertRCReturn(rc, rc);
5065	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5066	AssertRCReturn(rc, rc);
5067
5068	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5069	CPUMSetGuestCR4(pVCpu, uVal);
5070	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5071	}
5072	return rc;
5073	}
5074
5075
5076	/**
5077	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5078	*
5079	* @returns VBox status code.
5080	* @param pVCpu Pointer to the VMCPU.
5081	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5082	* out-of-sync. Make sure to update the required fields
5083	* before using them.
5084	*
5085	* @remarks No-long-jump zone!!!
5086	*/
5087	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5088	{
5089	int rc = VINF_SUCCESS;
5090	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5091	{
5092	uint64_t u64Val = 0;
5093	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5094	AssertRCReturn(rc, rc);
5095
5096	pMixedCtx->rip = u64Val;
5097	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5098	}
5099	return rc;
5100	}
5101
5102
5103	/**
5104	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5105	*
5106	* @returns VBox status code.
5107	* @param pVCpu Pointer to the VMCPU.
5108	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5109	* out-of-sync. Make sure to update the required fields
5110	* before using them.
5111	*
5112	* @remarks No-long-jump zone!!!
5113	*/
5114	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5115	{
5116	int rc = VINF_SUCCESS;
5117	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5118	{
5119	uint64_t u64Val = 0;
5120	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5121	AssertRCReturn(rc, rc);
5122
5123	pMixedCtx->rsp = u64Val;
5124	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5125	}
5126	return rc;
5127	}
5128
5129
5130	/**
5131	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5132	*
5133	* @returns VBox status code.
5134	* @param pVCpu Pointer to the VMCPU.
5135	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5136	* out-of-sync. Make sure to update the required fields
5137	* before using them.
5138	*
5139	* @remarks No-long-jump zone!!!
5140	*/
5141	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5142	{
5143	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5144	{
5145	uint32_t uVal = 0;
5146	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5147	AssertRCReturn(rc, rc);
5148
5149	pMixedCtx->eflags.u32 = uVal;
5150	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5151	{
5152	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5153	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5154
5155	pMixedCtx->eflags.Bits.u1VM = 0;
5156	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5157	}
5158
5159	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5160	}
5161	return VINF_SUCCESS;
5162	}
5163
5164
5165	/**
5166	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5167	* guest-CPU context.
5168	*/
5169	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5170	{
5171	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5172	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5173	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5174	return rc;
5175	}
5176
5177
5178	/**
5179	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5180	* from the guest-state area in the VMCS.
5181	*
5182	* @param pVCpu Pointer to the VMCPU.
5183	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5184	* out-of-sync. Make sure to update the required fields
5185	* before using them.
5186	*
5187	* @remarks No-long-jump zone!!!
5188	*/
5189	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5190	{
5191	uint32_t uIntrState = 0;
5192	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5193	AssertRC(rc);
5194
5195	if (!uIntrState)
5196	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5197	else
5198	{
5199	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5200	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5201	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5202	AssertRC(rc);
5203	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5204	AssertRC(rc);
5205
5206	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5207	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5208	}
5209	}
5210
5211
5212	/**
5213	* Saves the guest's activity state.
5214	*
5215	* @returns VBox status code.
5216	* @param pVCpu Pointer to the VMCPU.
5217	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5218	* out-of-sync. Make sure to update the required fields
5219	* before using them.
5220	*
5221	* @remarks No-long-jump zone!!!
5222	*/
5223	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5224	{
5225	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5226	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5227	return VINF_SUCCESS;
5228	}
5229
5230
5231	/**
5232	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5233	* the current VMCS into the guest-CPU context.
5234	*
5235	* @returns VBox status code.
5236	* @param pVCpu Pointer to the VMCPU.
5237	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5238	* out-of-sync. Make sure to update the required fields
5239	* before using them.
5240	*
5241	* @remarks No-long-jump zone!!!
5242	*/
5243	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5244	{
5245	int rc = VINF_SUCCESS;
5246	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5247	{
5248	uint32_t u32Val = 0;
5249	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5250	pMixedCtx->SysEnter.cs = u32Val;
5251	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5252	}
5253
5254	uint64_t u64Val = 0;
5255	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5256	{
5257	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5258	pMixedCtx->SysEnter.eip = u64Val;
5259	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5260	}
5261	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5262	{
5263	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5264	pMixedCtx->SysEnter.esp = u64Val;
5265	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5266	}
5267	return rc;
5268	}
5269
5270
5271	/**
5272	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5273	* context.
5274	*
5275	* @returns VBox status code.
5276	* @param pVCpu Pointer to the VMCPU.
5277	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5278	* out-of-sync. Make sure to update the required fields
5279	* before using them.
5280	*
5281	* @remarks No-long-jump zone!!!
5282	*/
5283	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5284	{
5285	int rc = VINF_SUCCESS;
5286	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5287	{
5288	uint64_t u64Val = 0;
5289	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5290	pMixedCtx->fs.u64Base = u64Val;
5291	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5292	}
5293	return rc;
5294	}
5295
5296
5297	/**
5298	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5299	* context.
5300	*
5301	* @returns VBox status code.
5302	* @param pVCpu Pointer to the VMCPU.
5303	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5304	* out-of-sync. Make sure to update the required fields
5305	* before using them.
5306	*
5307	* @remarks No-long-jump zone!!!
5308	*/
5309	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5310	{
5311	int rc = VINF_SUCCESS;
5312	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5313	{
5314	uint64_t u64Val = 0;
5315	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5316	pMixedCtx->gs.u64Base = u64Val;
5317	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5318	}
5319	return rc;
5320	}
5321
5322
5323	/**
5324	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5325	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5326	* and TSC_AUX.
5327	*
5328	* @returns VBox status code.
5329	* @param pVCpu Pointer to the VMCPU.
5330	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5331	* out-of-sync. Make sure to update the required fields
5332	* before using them.
5333	*
5334	* @remarks No-long-jump zone!!!
5335	*/
5336	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5337	{
5338	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5339	return VINF_SUCCESS;
5340
5341	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5342	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5343	{
5344	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5345	pMsr += i;
5346	switch (pMsr->u32IndexMSR)
5347	{
5348	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5349	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5350	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5351	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5352	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5353	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5354	default:
5355	{
5356	AssertFailed();
5357	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5358	}
5359	}
5360	}
5361	#endif
5362
5363	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5364	return VINF_SUCCESS;
5365	}
5366
5367
5368	/**
5369	* Saves the guest control registers from the current VMCS into the guest-CPU
5370	* context.
5371	*
5372	* @returns VBox status code.
5373	* @param pVCpu Pointer to the VMCPU.
5374	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5375	* out-of-sync. Make sure to update the required fields
5376	* before using them.
5377	*
5378	* @remarks No-long-jump zone!!!
5379	*/
5380	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5381	{
5382	/* Guest CR0. Guest FPU. */
5383	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5384	AssertRCReturn(rc, rc);
5385
5386	/* Guest CR4. */
5387	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5388	AssertRCReturn(rc, rc);
5389
5390	/* Guest CR2 - updated always during the world-switch or in #PF. */
5391	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5392	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5393	{
5394	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5395	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5396
5397	PVM pVM = pVCpu->CTX_SUFF(pVM);
5398	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5399	\|\| ( pVM->hm.s.fNestedPaging
5400	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5401	{
5402	uint64_t u64Val = 0;
5403	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5404	if (pMixedCtx->cr3 != u64Val)
5405	{
5406	CPUMSetGuestCR3(pVCpu, u64Val);
5407	if (VMMRZCallRing3IsEnabled(pVCpu))
5408	{
5409	PGMUpdateCR3(pVCpu, u64Val);
5410	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5411	}
5412	else
5413	{
5414	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5415	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5416	}
5417	}
5418
5419	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5420	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5421	{
5422	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5423	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5424	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5425	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5426
5427	if (VMMRZCallRing3IsEnabled(pVCpu))
5428	{
5429	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5430	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5431	}
5432	else
5433	{
5434	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5435	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5436	}
5437	}
5438	}
5439
5440	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5441	}
5442
5443	/*
5444	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5445	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5446	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5447	*
5448	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5449	*/
5450	if (VMMRZCallRing3IsEnabled(pVCpu))
5451	{
5452	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5453	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5454
5455	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5456	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5457
5458	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5459	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5460	}
5461
5462	return rc;
5463	}
5464
5465
5466	/**
5467	* Reads a guest segment register from the current VMCS into the guest-CPU
5468	* context.
5469	*
5470	* @returns VBox status code.
5471	* @param pVCpu Pointer to the VMCPU.
5472	* @param idxSel Index of the selector in the VMCS.
5473	* @param idxLimit Index of the segment limit in the VMCS.
5474	* @param idxBase Index of the segment base in the VMCS.
5475	* @param idxAccess Index of the access rights of the segment in the VMCS.
5476	* @param pSelReg Pointer to the segment selector.
5477	*
5478	* @remarks No-long-jump zone!!!
5479	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5480	* macro as that takes care of whether to read from the VMCS cache or
5481	* not.
5482	*/
5483	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5484	PCPUMSELREG pSelReg)
5485	{
5486	uint32_t u32Val = 0;
5487	int rc = VMXReadVmcs32(idxSel, &u32Val);
5488	AssertRCReturn(rc, rc);
5489	pSelReg->Sel = (uint16_t)u32Val;
5490	pSelReg->ValidSel = (uint16_t)u32Val;
5491	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5492
5493	rc = VMXReadVmcs32(idxLimit, &u32Val);
5494	AssertRCReturn(rc, rc);
5495	pSelReg->u32Limit = u32Val;
5496
5497	uint64_t u64Val = 0;
5498	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5499	AssertRCReturn(rc, rc);
5500	pSelReg->u64Base = u64Val;
5501
5502	rc = VMXReadVmcs32(idxAccess, &u32Val);
5503	AssertRCReturn(rc, rc);
5504	pSelReg->Attr.u = u32Val;
5505
5506	/*
5507	* If VT-x marks the segment as unusable, most other bits remain undefined:
5508	* - For CS the L, D and G bits have meaning.
5509	* - For SS the DPL have meaning (it -is- the CPL for Intel and VBox).
5510	* - For the remaining data segments no bits are defined.
5511	*
5512	* The present bit and the unused bit has been observed to be set at the
5513	* same time (the selector was supposed to invalid as we started executing
5514	* a V8086 interrupt in ring-0).
5515	*
5516	* What should be important for the rest of the VBox code that the P bit is
5517	* cleared. Some of the other VBox code recognizes the unusable bit, but
5518	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5519	* safe side here we'll strip off P and other bits we don't care about. If
5520	* any code breaks because attr.u != 0 when Sel < 4, it should be fixed.
5521	*
5522	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5523	*/
5524	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5525	{
5526	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5527
5528	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5529	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5530	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5531	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5532	#ifdef DEBUG_bird
5533	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5534	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5535	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5536	#endif
5537	}
5538	return VINF_SUCCESS;
5539	}
5540
5541
5542	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5543	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5544	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5545	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5546	#else
5547	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5548	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5549	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5550	#endif
5551
5552
5553	/**
5554	* Saves the guest segment registers from the current VMCS into the guest-CPU
5555	* context.
5556	*
5557	* @returns VBox status code.
5558	* @param pVCpu Pointer to the VMCPU.
5559	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5560	* out-of-sync. Make sure to update the required fields
5561	* before using them.
5562	*
5563	* @remarks No-long-jump zone!!!
5564	*/
5565	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5566	{
5567	/* Guest segment registers. */
5568	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5569	{
5570	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5571	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5572	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5573	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5574	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5575	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5576	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5577
5578	/* Restore segment attributes for real-on-v86 mode hack. */
5579	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5580	{
5581	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5582	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5583	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5584	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5585	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5586	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5587	}
5588	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5589	}
5590
5591	return VINF_SUCCESS;
5592	}
5593
5594
5595	/**
5596	* Saves the guest descriptor table registers and task register from the current
5597	* VMCS into the guest-CPU context.
5598	*
5599	* @returns VBox status code.
5600	* @param pVCpu Pointer to the VMCPU.
5601	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5602	* out-of-sync. Make sure to update the required fields
5603	* before using them.
5604	*
5605	* @remarks No-long-jump zone!!!
5606	*/
5607	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5608	{
5609	int rc = VINF_SUCCESS;
5610
5611	/* Guest LDTR. */
5612	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5613	{
5614	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5615	AssertRCReturn(rc, rc);
5616	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5617	}
5618
5619	/* Guest GDTR. */
5620	uint64_t u64Val = 0;
5621	uint32_t u32Val = 0;
5622	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5623	{
5624	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5625	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5626	pMixedCtx->gdtr.pGdt = u64Val;
5627	pMixedCtx->gdtr.cbGdt = u32Val;
5628	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5629	}
5630
5631	/* Guest IDTR. */
5632	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5633	{
5634	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5635	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5636	pMixedCtx->idtr.pIdt = u64Val;
5637	pMixedCtx->idtr.cbIdt = u32Val;
5638	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5639	}
5640
5641	/* Guest TR. */
5642	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5643	{
5644	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5645	AssertRCReturn(rc, rc);
5646
5647	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5648	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5649	{
5650	rc = VMXLOCAL_READ_SEG(TR, tr);
5651	AssertRCReturn(rc, rc);
5652	}
5653	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5654	}
5655	return rc;
5656	}
5657
5658	#undef VMXLOCAL_READ_SEG
5659
5660
5661	/**
5662	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5663	* context.
5664	*
5665	* @returns VBox status code.
5666	* @param pVCpu Pointer to the VMCPU.
5667	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5668	* out-of-sync. Make sure to update the required fields
5669	* before using them.
5670	*
5671	* @remarks No-long-jump zone!!!
5672	*/
5673	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5674	{
5675	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5676	{
5677	if (!CPUMIsHyperDebugStateActive(pVCpu))
5678	{
5679	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5680	uint32_t u32Val;
5681	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5682	pMixedCtx->dr[7] = u32Val;
5683	}
5684
5685	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5686	}
5687	return VINF_SUCCESS;
5688	}
5689
5690
5691	/**
5692	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5693	*
5694	* @returns VBox status code.
5695	* @param pVCpu Pointer to the VMCPU.
5696	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5697	* out-of-sync. Make sure to update the required fields
5698	* before using them.
5699	*
5700	* @remarks No-long-jump zone!!!
5701	*/
5702	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5703	{
5704	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5705	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5706	return VINF_SUCCESS;
5707	}
5708
5709
5710	/**
5711	* Saves the entire guest state from the currently active VMCS into the
5712	* guest-CPU context. This essentially VMREADs all guest-data.
5713	*
5714	* @returns VBox status code.
5715	* @param pVCpu Pointer to the VMCPU.
5716	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5717	* out-of-sync. Make sure to update the required fields
5718	* before using them.
5719	*/
5720	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5721	{
5722	Assert(pVCpu);
5723	Assert(pMixedCtx);
5724
5725	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5726	return VINF_SUCCESS;
5727
5728	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5729	there is no real need to. */
5730	if (VMMRZCallRing3IsEnabled(pVCpu))
5731	VMMR0LogFlushDisable(pVCpu);
5732	else
5733	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5734	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5735
5736	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5737	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5738
5739	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5740	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5741
5742	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5743	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5744
5745	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5746	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5747
5748	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5749	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5750
5751	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5752	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5753
5754	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5755	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5756
5757	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5758	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5759
5760	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5761	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5762
5763	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5764	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5765
5766	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5767	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5768
5769	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5770	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5771
5772	if (VMMRZCallRing3IsEnabled(pVCpu))
5773	VMMR0LogFlushEnable(pVCpu);
5774
5775	return rc;
5776	}
5777
5778
5779	/**
5780	* Check per-VM and per-VCPU force flag actions that require us to go back to
5781	* ring-3 for one reason or another.
5782	*
5783	* @returns VBox status code (information status code included).
5784	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5785	* ring-3.
5786	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5787	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5788	* interrupts)
5789	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5790	* all EMTs to be in ring-3.
5791	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5792	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5793	* to the EM loop.
5794	*
5795	* @param pVM Pointer to the VM.
5796	* @param pVCpu Pointer to the VMCPU.
5797	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5798	* out-of-sync. Make sure to update the required fields
5799	* before using them.
5800	*/
5801	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5802	{
5803	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5804
5805	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5806	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5807	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5808	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5809	{
5810	/* We need the control registers now, make sure the guest-CPU context is updated. */
5811	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5812	AssertRCReturn(rc3, rc3);
5813
5814	/* Pending HM CR3 sync. */
5815	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5816	{
5817	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5818	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
5819	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5820	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5821	}
5822
5823	/* Pending HM PAE PDPEs. */
5824	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5825	{
5826	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5827	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5828	}
5829
5830	/* Pending PGM C3 sync. */
5831	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5832	{
5833	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5834	if (rc2 != VINF_SUCCESS)
5835	{
5836	AssertRC(rc2);
5837	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5838	return rc2;
5839	}
5840	}
5841
5842	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5843	/* -XXX- what was that about single stepping? */
5844	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5845	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5846	{
5847	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5848	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5849	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5850	return rc2;
5851	}
5852
5853	/* Pending VM request packets, such as hardware interrupts. */
5854	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5855	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5856	{
5857	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5858	return VINF_EM_PENDING_REQUEST;
5859	}
5860
5861	/* Pending PGM pool flushes. */
5862	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5863	{
5864	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5865	return VINF_PGM_POOL_FLUSH_PENDING;
5866	}
5867
5868	/* Pending DMA requests. */
5869	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5870	{
5871	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5872	return VINF_EM_RAW_TO_R3;
5873	}
5874	}
5875
5876	/* Paranoia. */
5877	return VINF_SUCCESS;
5878	}
5879
5880
5881	/**
5882	* Converts any TRPM trap into a pending HM event. This is typically used when
5883	* entering from ring-3 (not longjmp returns).
5884	*
5885	* @param pVCpu Pointer to the VMCPU.
5886	*/
5887	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5888	{
5889	Assert(TRPMHasTrap(pVCpu));
5890	Assert(!pVCpu->hm.s.Event.fPending);
5891
5892	uint8_t uVector;
5893	TRPMEVENT enmTrpmEvent;
5894	RTGCUINT uErrCode;
5895	RTGCUINTPTR GCPtrFaultAddress;
5896	uint8_t cbInstr;
5897
5898	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5899	AssertRC(rc);
5900
5901	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5902	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5903	if (enmTrpmEvent == TRPM_TRAP)
5904	{
5905	switch (uVector)
5906	{
5907	case X86_XCPT_BP:
5908	case X86_XCPT_OF:
5909	{
5910	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5911	break;
5912	}
5913
5914	case X86_XCPT_PF:
5915	case X86_XCPT_DF:
5916	case X86_XCPT_TS:
5917	case X86_XCPT_NP:
5918	case X86_XCPT_SS:
5919	case X86_XCPT_GP:
5920	case X86_XCPT_AC:
5921	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5922	/* no break! */
5923	default:
5924	{
5925	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5926	break;
5927	}
5928	}
5929	}
5930	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5931	{
5932	if (uVector == X86_XCPT_NMI)
5933	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5934	else
5935	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5936	}
5937	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5938	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5939	else
5940	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5941
5942	rc = TRPMResetTrap(pVCpu);
5943	AssertRC(rc);
5944	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5945	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5946
5947	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5948	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
5949	}
5950
5951
5952	/**
5953	* Converts any pending HM event into a TRPM trap. Typically used when leaving
5954	* VT-x to execute any instruction.
5955	*
5956	* @param pvCpu Pointer to the VMCPU.
5957	*/
5958	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5959	{
5960	Assert(pVCpu->hm.s.Event.fPending);
5961
5962	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5963	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5964	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5965	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5966
5967	/* If a trap was already pending, we did something wrong! */
5968	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5969
5970	TRPMEVENT enmTrapType;
5971	switch (uVectorType)
5972	{
5973	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5974	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5975	enmTrapType = TRPM_HARDWARE_INT;
5976	break;
5977	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5978	enmTrapType = TRPM_SOFTWARE_INT;
5979	break;
5980	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5981	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5982	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5983	enmTrapType = TRPM_TRAP;
5984	break;
5985	default:
5986	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5987	enmTrapType = TRPM_32BIT_HACK;
5988	break;
5989	}
5990
5991	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5992
5993	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5994	AssertRC(rc);
5995
5996	if (fErrorCodeValid)
5997	TRPMSetErrorCode(pVCpu, uErrorCode);
5998
5999	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6000	&& uVector == X86_XCPT_PF)
6001	{
6002	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6003	}
6004	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6005	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6006	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6007	{
6008	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6009	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6010	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6011	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6012	}
6013	pVCpu->hm.s.Event.fPending = false;
6014	}
6015
6016
6017	/**
6018	* Does the necessary state syncing before returning to ring-3 for any reason
6019	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6020	*
6021	* @param pVM Pointer to the VM.
6022	* @param pVCpu Pointer to the VMCPU.
6023	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6024	* out-of-sync. Make sure to update the required fields
6025	* before using them.
6026	*
6027	* @remarks No-long-jmp zone!!!
6028	*/
6029	static void hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6030	{
6031	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6032	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6033
6034	/* Save the guest state if necessary. */
6035	if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6036	{
6037	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6038	AssertRC(rc);
6039	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6040	}
6041
6042	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6043	if (CPUMIsGuestFPUStateActive(pVCpu))
6044	{
6045	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6046	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6047	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
6048	}
6049
6050	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6051	#ifdef VBOX_STRICT
6052	if (CPUMIsHyperDebugStateActive(pVCpu))
6053	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6054	#endif
6055	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6056	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
6057	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6058	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6059
6060	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6061	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6062	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6063	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6064	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6065	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6066	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6067	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6068
6069	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6070	}
6071
6072
6073	/**
6074	* Does the necessary state syncing before doing a longjmp to ring-3.
6075	*
6076	* @param pVM Pointer to the VM.
6077	* @param pVCpu Pointer to the VMCPU.
6078	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6079	* out-of-sync. Make sure to update the required fields
6080	* before using them.
6081	*
6082	* @remarks No-long-jmp zone!!!
6083	*/
6084	DECLINLINE(void) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6085	{
6086	hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6087	}
6088
6089
6090	/**
6091	* Take necessary actions before going back to ring-3.
6092	*
6093	* An action requires us to go back to ring-3. This function does the necessary
6094	* steps before we can safely return to ring-3. This is not the same as longjmps
6095	* to ring-3, this is voluntary and prepares the guest so it may continue
6096	* executing outside HM (recompiler/IEM).
6097	*
6098	* @param pVM Pointer to the VM.
6099	* @param pVCpu Pointer to the VMCPU.
6100	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6101	* out-of-sync. Make sure to update the required fields
6102	* before using them.
6103	* @param rcExit The reason for exiting to ring-3. Can be
6104	* VINF_VMM_UNKNOWN_RING3_CALL.
6105	*/
6106	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6107	{
6108	Assert(pVM);
6109	Assert(pVCpu);
6110	Assert(pMixedCtx);
6111	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6112
6113	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6114	{
6115	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
6116	return;
6117	}
6118	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6119	{
6120	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6121	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6122	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6123	pVCpu->hm.s.vmx.LastError.idCurrentCpu = RTMpCpuId();
6124	return;
6125	}
6126
6127	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6128	VMMRZCallRing3Disable(pVCpu);
6129	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6130
6131	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6132	if (pVCpu->hm.s.Event.fPending)
6133	{
6134	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6135	Assert(!pVCpu->hm.s.Event.fPending);
6136	}
6137
6138	/* Save guest state and restore host state bits. */
6139	hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6140	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6141
6142	/* Sync recompiler state. */
6143	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6144	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6145	\| CPUM_CHANGED_LDTR
6146	\| CPUM_CHANGED_GDTR
6147	\| CPUM_CHANGED_IDTR
6148	\| CPUM_CHANGED_TR
6149	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6150	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6151	if ( pVM->hm.s.fNestedPaging
6152	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6153	{
6154	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6155	}
6156
6157	/* On our way back from ring-3 the following needs to be done. */
6158	/** @todo This can change with preemption hooks. */
6159	if (rcExit == VINF_EM_RAW_INTERRUPT)
6160	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
6161	else
6162	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
6163
6164	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6165	VMMRZCallRing3Enable(pVCpu);
6166	}
6167
6168
6169	/**
6170	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6171	* longjump to ring-3 and possibly get preempted.
6172	*
6173	* @param pVCpu Pointer to the VMCPU.
6174	* @param enmOperation The operation causing the ring-3 longjump.
6175	* @param pvUser The user argument (pointer to the possibly
6176	* out-of-date guest-CPU context).
6177	*
6178	* @remarks Must never be called with @a enmOperation ==
6179	* VMMCALLRING3_VM_R0_ASSERTION.
6180	*/
6181	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6182	{
6183	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6184	Assert(pVCpu);
6185	Assert(pvUser);
6186	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6187	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6188
6189	VMMRZCallRing3Disable(pVCpu);
6190	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6191	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6192	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6193	VMMRZCallRing3Enable(pVCpu);
6194	}
6195
6196
6197	/**
6198	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6199	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6200	*
6201	* @param pVCpu Pointer to the VMCPU.
6202	*/
6203	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6204	{
6205	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6206	{
6207	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6208	{
6209	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6210	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6211	AssertRC(rc);
6212	}
6213	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6214	}
6215
6216
6217	/**
6218	* Injects any pending events into the guest if the guest is in a state to
6219	* receive them.
6220	*
6221	* @returns VBox status code (informational status codes included).
6222	* @param pVCpu Pointer to the VMCPU.
6223	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6224	* out-of-sync. Make sure to update the required fields
6225	* before using them.
6226	*/
6227	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6228	{
6229	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6230	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6231	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6232	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6233
6234	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6235	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6236	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6237	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6238	Assert(!TRPMHasTrap(pVCpu));
6239
6240	int rc = VINF_SUCCESS;
6241	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6242	{
6243	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6244	bool fInject = true;
6245	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6246	{
6247	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6248	AssertRCReturn(rc, rc);
6249	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6250	if ( fBlockInt
6251	\|\| fBlockSti
6252	\|\| fBlockMovSS)
6253	{
6254	fInject = false;
6255	}
6256	}
6257	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6258	&& ( fBlockMovSS
6259	\|\| fBlockSti))
6260	{
6261	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6262	fInject = false;
6263	}
6264
6265	if (fInject)
6266	{
6267	Log4(("Injecting pending event vcpu[%RU32]\n", pVCpu->idCpu));
6268	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6269	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6270	AssertRCReturn(rc, rc);
6271	pVCpu->hm.s.Event.fPending = false;
6272
6273	#ifdef VBOX_WITH_STATISTICS
6274	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6275	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6276	else
6277	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6278	#endif
6279	}
6280	else
6281	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6282	} /** @todo SMI. SMIs take priority over NMIs. */
6283	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6284	{
6285	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6286	if ( !fBlockMovSS
6287	&& !fBlockSti)
6288	{
6289	Log4(("Injecting NMI\n"));
6290	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6291	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6292	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6293	0 /* GCPtrFaultAddress */, &uIntrState);
6294	AssertRCReturn(rc, rc);
6295	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6296
6297	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6298	}
6299	else
6300	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6301	}
6302	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6303	&& !pVCpu->hm.s.fSingleInstruction)
6304	{
6305	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6306	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6307	AssertRCReturn(rc, rc);
6308	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6309	if ( !fBlockInt
6310	&& !fBlockSti
6311	&& !fBlockMovSS)
6312	{
6313	uint8_t u8Interrupt;
6314	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6315	if (RT_SUCCESS(rc))
6316	{
6317	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6318	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6319	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6320	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6321	0 /* GCPtrFaultAddress */, &uIntrState);
6322
6323	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6324	}
6325	else
6326	{
6327	/** @todo Does this actually happen? If not turn it into an assertion. */
6328	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6329	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6330	rc = VINF_SUCCESS;
6331	}
6332	}
6333	else
6334	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6335	}
6336
6337	/*
6338	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6339	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6340	*/
6341	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6342	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6343	int rc2 = VINF_SUCCESS;
6344	if ( fBlockSti
6345	\|\| fBlockMovSS)
6346	{
6347	if (!DBGFIsStepping(pVCpu))
6348	{
6349	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6350	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6351	{
6352	/*
6353	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6354	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6355	* See Intel spec. 27.3.4 "Saving Non-Register State".
6356	*/
6357	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6358	AssertRCReturn(rc, rc);
6359	}
6360	}
6361	else
6362	{
6363	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6364	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6365	uIntrState = 0;
6366	}
6367	}
6368
6369	/*
6370	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6371	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6372	*/
6373	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6374	AssertRC(rc2);
6375
6376	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6377	return rc;
6378	}
6379
6380
6381	/**
6382	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6383	*
6384	* @param pVCpu Pointer to the VMCPU.
6385	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6386	* out-of-sync. Make sure to update the required fields
6387	* before using them.
6388	*/
6389	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6390	{
6391	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6392	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6393	}
6394
6395
6396	/**
6397	* Injects a double-fault (#DF) exception into the VM.
6398	*
6399	* @returns VBox status code (informational status code included).
6400	* @param pVCpu Pointer to the VMCPU.
6401	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6402	* out-of-sync. Make sure to update the required fields
6403	* before using them.
6404	*/
6405	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6406	{
6407	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6408	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6409	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6410	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6411	puIntrState);
6412	}
6413
6414
6415	/**
6416	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6417	*
6418	* @param pVCpu Pointer to the VMCPU.
6419	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6420	* out-of-sync. Make sure to update the required fields
6421	* before using them.
6422	*/
6423	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6424	{
6425	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6426	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6427	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6428	}
6429
6430
6431	/**
6432	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6433	*
6434	* @param pVCpu Pointer to the VMCPU.
6435	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6436	* out-of-sync. Make sure to update the required fields
6437	* before using them.
6438	* @param cbInstr The value of RIP that is to be pushed on the guest
6439	* stack.
6440	*/
6441	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6442	{
6443	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6444	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6445	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6446	}
6447
6448
6449	/**
6450	* Injects a general-protection (#GP) fault into the VM.
6451	*
6452	* @returns VBox status code (informational status code included).
6453	* @param pVCpu Pointer to the VMCPU.
6454	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6455	* out-of-sync. Make sure to update the required fields
6456	* before using them.
6457	* @param u32ErrorCode The error code associated with the #GP.
6458	*/
6459	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6460	uint32_t *puIntrState)
6461	{
6462	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6463	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6464	if (fErrorCodeValid)
6465	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6466	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6467	puIntrState);
6468	}
6469
6470
6471	/**
6472	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6473	*
6474	* @param pVCpu Pointer to the VMCPU.
6475	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6476	* out-of-sync. Make sure to update the required fields
6477	* before using them.
6478	* @param uVector The software interrupt vector number.
6479	* @param cbInstr The value of RIP that is to be pushed on the guest
6480	* stack.
6481	*/
6482	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6483	{
6484	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6485	if ( uVector == X86_XCPT_BP
6486	\|\| uVector == X86_XCPT_OF)
6487	{
6488	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6489	}
6490	else
6491	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6492	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6493	}
6494
6495
6496	/**
6497	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6498	* stack.
6499	*
6500	* @returns VBox status code (information status code included).
6501	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6502	* @param pVM Pointer to the VM.
6503	* @param pMixedCtx Pointer to the guest-CPU context.
6504	* @param uValue The value to push to the guest stack.
6505	*/
6506	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6507	{
6508	/*
6509	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6510	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6511	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6512	*/
6513	if (pMixedCtx->sp == 1)
6514	return VINF_EM_RESET;
6515	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6516	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6517	AssertRCReturn(rc, rc);
6518	return rc;
6519	}
6520
6521
6522	/**
6523	* Injects an event into the guest upon VM-entry by updating the relevant fields
6524	* in the VM-entry area in the VMCS.
6525	*
6526	* @returns VBox status code (informational error codes included).
6527	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6528	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6529	*
6530	* @param pVCpu Pointer to the VMCPU.
6531	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6532	* be out-of-sync. Make sure to update the required
6533	* fields before using them.
6534	* @param u64IntrInfo The VM-entry interruption-information field.
6535	* @param cbInstr The VM-entry instruction length in bytes (for
6536	* software interrupts, exceptions and privileged
6537	* software exceptions).
6538	* @param u32ErrCode The VM-entry exception error code.
6539	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6540	* @param puIntrState Pointer to the current guest interruptibility-state.
6541	* This interruptibility-state will be updated if
6542	* necessary. This cannot not be NULL.
6543	*
6544	* @remarks No-long-jump zone!!!
6545	* @remarks Requires CR0!
6546	*/
6547	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6548	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6549	{
6550	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6551	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6552	Assert(puIntrState);
6553	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6554
6555	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6556	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6557
6558	#ifdef VBOX_STRICT
6559	/* Validate the error-code-valid bit for hardware exceptions. */
6560	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6561	{
6562	switch (uVector)
6563	{
6564	case X86_XCPT_PF:
6565	case X86_XCPT_DF:
6566	case X86_XCPT_TS:
6567	case X86_XCPT_NP:
6568	case X86_XCPT_SS:
6569	case X86_XCPT_GP:
6570	case X86_XCPT_AC:
6571	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6572	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6573	/* fallthru */
6574	default:
6575	break;
6576	}
6577	}
6578	#endif
6579
6580	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6581	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6582	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6583
6584	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6585
6586	/* We require CR0 to check if the guest is in real-mode. */
6587	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6588	AssertRCReturn(rc, rc);
6589
6590	/*
6591	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6592	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6593	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6594	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6595	*/
6596	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6597	{
6598	PVM pVM = pVCpu->CTX_SUFF(pVM);
6599	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6600	{
6601	Assert(PDMVmmDevHeapIsEnabled(pVM));
6602	Assert(pVM->hm.s.vmx.pRealModeTSS);
6603
6604	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6605	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6606	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6607	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6608	AssertRCReturn(rc, rc);
6609	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6610
6611	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6612	const size_t cbIdtEntry = 4;
6613	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6614	{
6615	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6616	if (uVector == X86_XCPT_DF)
6617	return VINF_EM_RESET;
6618	else if (uVector == X86_XCPT_GP)
6619	{
6620	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6621	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6622	}
6623
6624	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6625	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6626	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6627	}
6628
6629	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6630	uint16_t uGuestIp = pMixedCtx->ip;
6631	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6632	{
6633	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6634	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6635	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6636	}
6637	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6638	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6639
6640	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6641	uint16_t offIdtEntry = 0;
6642	RTSEL selIdtEntry = 0;
6643	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6644	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6645	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6646	AssertRCReturn(rc, rc);
6647
6648	/* Construct the stack frame for the interrupt/exception handler. */
6649	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6650	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6651	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6652	AssertRCReturn(rc, rc);
6653
6654	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6655	if (rc == VINF_SUCCESS)
6656	{
6657	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6658	pMixedCtx->rip = offIdtEntry;
6659	pMixedCtx->cs.Sel = selIdtEntry;
6660	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6661	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6662	&& uVector == X86_XCPT_PF)
6663	{
6664	pMixedCtx->cr2 = GCPtrFaultAddress;
6665	}
6666	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6667	\| HM_CHANGED_GUEST_RIP
6668	\| HM_CHANGED_GUEST_RFLAGS
6669	\| HM_CHANGED_GUEST_RSP;
6670
6671	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6672	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6673	{
6674	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6675	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6676	Log4(("Clearing inhibition due to STI.\n"));
6677	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6678	}
6679	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6680	}
6681	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6682	return rc;
6683	}
6684	else
6685	{
6686	/*
6687	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6688	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6689	*/
6690	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6691	}
6692	}
6693
6694	/* Validate. */
6695	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6696	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6697	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6698
6699	/* Inject. */
6700	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6701	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6702	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6703	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6704
6705	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6706	&& uVector == X86_XCPT_PF)
6707	{
6708	pMixedCtx->cr2 = GCPtrFaultAddress;
6709	}
6710
6711	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6712	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6713
6714	AssertRCReturn(rc, rc);
6715	return rc;
6716	}
6717
6718
6719	/**
6720	* Enters the VT-x session.
6721	*
6722	* @returns VBox status code.
6723	* @param pVM Pointer to the VM.
6724	* @param pVCpu Pointer to the VMCPU.
6725	* @param pCpu Pointer to the CPU info struct.
6726	*/
6727	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6728	{
6729	AssertPtr(pVM);
6730	AssertPtr(pVCpu);
6731	Assert(pVM->hm.s.vmx.fSupported);
6732	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6733	NOREF(pCpu);
6734
6735	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6736
6737	#ifdef VBOX_STRICT
6738	/* Make sure we're in VMX root mode. */
6739	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6740	if (!(u32HostCR4 & X86_CR4_VMXE))
6741	{
6742	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6743	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6744	}
6745	#endif
6746
6747	/* Load the active VMCS as the current one. */
6748	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6749	if (RT_FAILURE(rc))
6750	return rc;
6751
6752	/** @todo this will change with preemption hooks where can VMRESUME as long
6753	* as we're no preempted. */
6754	pVCpu->hm.s.fResumeVM = false;
6755	return VINF_SUCCESS;
6756	}
6757
6758
6759	/**
6760	* Leaves the VT-x session.
6761	*
6762	* @returns VBox status code.
6763	* @param pVM Pointer to the VM.
6764	* @param pVCpu Pointer to the VMCPU.
6765	* @param pCtx Pointer to the guest-CPU context.
6766	*/
6767	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6768	{
6769	AssertPtr(pVCpu);
6770	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6771	NOREF(pVM);
6772	NOREF(pCtx);
6773
6774	/** @todo this will change with preemption hooks where we only VMCLEAR when
6775	* we are actually going to be preempted, not all the time like we
6776	* currently do. */
6777
6778	/* Restore host-state bits that VT-x only restores partially. */
6779	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6780	{
6781	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6782	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6783	}
6784
6785	/*
6786	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6787	* and mark the VMCS launch-state as "clear".
6788	*/
6789	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6790	return rc;
6791	}
6792
6793
6794	/**
6795	* Saves the host state in the VMCS host-state.
6796	* Sets up the VM-exit MSR-load area.
6797	*
6798	* The CPU state will be loaded from these fields on every successful VM-exit.
6799	*
6800	* @returns VBox status code.
6801	* @param pVM Pointer to the VM.
6802	* @param pVCpu Pointer to the VMCPU.
6803	*
6804	* @remarks No-long-jump zone!!!
6805	*/
6806	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6807	{
6808	AssertPtr(pVM);
6809	AssertPtr(pVCpu);
6810	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6811
6812	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6813
6814	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6815	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6816	return VINF_SUCCESS;
6817
6818	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6819	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6820
6821	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6822	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6823
6824	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6825	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6826
6827	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6828	return rc;
6829	}
6830
6831
6832	/**
6833	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6834	* loaded from these fields on every successful VM-entry.
6835	*
6836	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6837	* Sets up the VM-entry controls.
6838	* Sets up the appropriate VMX non-root function to execute guest code based on
6839	* the guest CPU mode.
6840	*
6841	* @returns VBox status code.
6842	* @param pVM Pointer to the VM.
6843	* @param pVCpu Pointer to the VMCPU.
6844	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6845	* out-of-sync. Make sure to update the required fields
6846	* before using them.
6847	*
6848	* @remarks No-long-jump zone!!!
6849	*/
6850	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6851	{
6852	AssertPtr(pVM);
6853	AssertPtr(pVCpu);
6854	AssertPtr(pMixedCtx);
6855	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6856
6857	#ifdef LOG_ENABLED
6858	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
6859	* probably not initialized yet? Anyway this will do for now. */
6860	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
6861	VMMR0LogFlushDisable(pVCpu);
6862	#endif
6863
6864	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6865
6866	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6867
6868	/* Determine real-on-v86 mode. */
6869	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6870	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6871	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6872	{
6873	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6874	}
6875
6876	/*
6877	* Load the guest-state into the VMCS.
6878	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6879	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6880	*/
6881	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6882	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6883
6884	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6885	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6886
6887	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6888	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6889
6890	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6891	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6892
6893	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6894	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6895	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6896
6897	rc = hmR0VmxLoadGuestDebugState(pVCpu, pMixedCtx);
6898	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugState: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6899
6900	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6901	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6902
6903	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6904	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6905
6906	/* Must be done after hmR0VmxLoadGuestDebugState() as it may have updated eflags.TF for debugging purposes. */
6907	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6908	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6909
6910	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6911	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6912
6913	/* Clear any unused and reserved bits. */
6914	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
6915
6916	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6917	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
6918	pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
6919
6920	#ifdef LOG_ENABLED
6921	/* Only reenable log-flushing if the caller has it enabled. */
6922	if (!fCallerDisabledLogFlush)
6923	VMMR0LogFlushEnable(pVCpu);
6924	#endif
6925
6926	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6927	return rc;
6928	}
6929
6930
6931	/**
6932	* Loads the guest state into the VMCS guest-state area.
6933	*
6934	* @returns VBox status code.
6935	* @param pVM Pointer to the VM.
6936	* @param pVCpu Pointer to the VMCPU.
6937	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6938	* out-of-sync. Make sure to update the required fields
6939	* before using them.
6940	*
6941	* @remarks No-long-jump zone!!!
6942	*/
6943	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6944	{
6945	/*
6946	* Avoid reloading the guest state on longjmp reentrants and do it lazily just before executing the guest.
6947	* This only helps when we get rescheduled more than once to a different host CPU on a longjmp trip before
6948	* finally executing guest code.
6949	*/
6950	return VINF_SUCCESS;
6951	}
6952
6953
6954	/**
6955	* Does the preparations before executing guest code in VT-x.
6956	*
6957	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6958	* recompiler. We must be cautious what we do here regarding committing
6959	* guest-state information into the VMCS assuming we assuredly execute the
6960	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
6961	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6962	* that the recompiler can (and should) use them when it resumes guest
6963	* execution. Otherwise such operations must be done when we can no longer
6964	* exit to ring-3.
6965	*
6966	* @returns VBox status code (informational status codes included).
6967	* @retval VINF_SUCCESS if we can proceed with running the guest.
6968	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6969	* into the guest.
6970	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6971	*
6972	* @param pVM Pointer to the VM.
6973	* @param pVCpu Pointer to the VMCPU.
6974	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6975	* out-of-sync. Make sure to update the required fields
6976	* before using them.
6977	* @param pVmxTransient Pointer to the VMX transient structure.
6978	*
6979	* @remarks Called with preemption disabled.
6980	*/
6981	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6982	{
6983	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6984
6985	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6986	PGMRZDynMapFlushAutoSet(pVCpu);
6987	#endif
6988
6989	/* Check force flag actions that might require us to go back to ring-3. */
6990	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6991	if (rc != VINF_SUCCESS)
6992	return rc;
6993
6994	#ifndef IEM_VERIFICATION_MODE_FULL
6995	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6996	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6997	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6998	{
6999	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7000	RTGCPHYS GCPhysApicBase;
7001	GCPhysApicBase = pMixedCtx->msrApicBase;
7002	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7003
7004	/* Unalias any existing mapping. */
7005	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7006	AssertRCReturn(rc, rc);
7007
7008	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7009	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7010	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7011	AssertRCReturn(rc, rc);
7012
7013	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7014	}
7015	#endif /* !IEM_VERIFICATION_MODE_FULL */
7016
7017	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7018	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
7019	pVmxTransient->uEFlags = ASMIntDisableFlags();
7020	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7021	{
7022	ASMSetFlags(pVmxTransient->uEFlags);
7023	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7024	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
7025	return VINF_EM_RAW_INTERRUPT;
7026	}
7027	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7028	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7029	#endif
7030
7031	/*
7032	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
7033	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
7034	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
7035	*/
7036	/** @todo Rework event evaluation and injection to be completely separate. */
7037	if (TRPMHasTrap(pVCpu))
7038	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7039
7040	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7041	AssertRCReturn(rc, rc);
7042	return rc;
7043	}
7044
7045
7046	/**
7047	* Prepares to run guest code in VT-x and we've committed to doing so. This
7048	* means there is no backing out to ring-3 or anywhere else at this
7049	* point.
7050	*
7051	* @param pVM Pointer to the VM.
7052	* @param pVCpu Pointer to the VMCPU.
7053	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7054	* out-of-sync. Make sure to update the required fields
7055	* before using them.
7056	* @param pVmxTransient Pointer to the VMX transient structure.
7057	*
7058	* @remarks Called with preemption disabled.
7059	* @remarks No-long-jump zone!!!
7060	*/
7061	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7062	{
7063	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7064	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7065
7066	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7067	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
7068	pVmxTransient->uEFlags = ASMIntDisableFlags();
7069	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7070	#endif
7071
7072	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
7073	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7074	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7075	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7076	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
7077	#endif
7078	int rc = VINF_SUCCESS;
7079	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7080	{
7081	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7082	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7083	}
7084	else if (pVCpu->hm.s.fContextUseFlags)
7085	{
7086	rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7087	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7088	}
7089	AssertRC(rc);
7090	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
7091
7092	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7093	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7094	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7095	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7096	#endif
7097
7098	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
7099	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7100	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7101
7102	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7103	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
7104	{
7105	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7106	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7107	}
7108
7109	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7110	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
7111	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
7112
7113	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7114
7115	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7116	to start executing. */
7117
7118	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7119	/*
7120	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7121	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7122	*/
7123	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7124	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7125	{
7126	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7127	uint64_t u64HostTscAux = 0;
7128	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7129	AssertRC(rc2);
7130	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7131	}
7132	#endif
7133	}
7134
7135
7136	/**
7137	* Performs some essential restoration of state after running guest code in
7138	* VT-x.
7139	*
7140	* @param pVM Pointer to the VM.
7141	* @param pVCpu Pointer to the VMCPU.
7142	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7143	* out-of-sync. Make sure to update the required fields
7144	* before using them.
7145	* @param pVmxTransient Pointer to the VMX transient structure.
7146	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7147	*
7148	* @remarks Called with interrupts disabled.
7149	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7150	* unconditionally when it is safe to do so.
7151	*/
7152	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7153	{
7154	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7155
7156	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7157	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7158	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7159	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7160	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7161
7162	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7163	{
7164	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7165	/* Restore host's TSC_AUX. */
7166	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7167	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7168	#endif
7169	/** @todo Find a way to fix hardcoding a guestimate. */
7170	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7171	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7172	}
7173
7174	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7175	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7176	Assert(!(ASMGetFlags() & X86_EFL_IF));
7177	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7178
7179	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
7180	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7181
7182	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7183	uint32_t uExitReason;
7184	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7185	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7186	AssertRC(rc);
7187	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7188	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7189
7190	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
7191	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7192
7193	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7194	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7195	{
7196	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7197	pVmxTransient->fVMEntryFailed));
7198	return;
7199	}
7200
7201	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7202	{
7203	/* Update the guest interruptibility-state from the VMCS. */
7204	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7205	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7206	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7207	AssertRC(rc);
7208	#endif
7209	/*
7210	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7211	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7212	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7213	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7214	*/
7215	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7216	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7217	{
7218	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7219	AssertRC(rc);
7220	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7221	}
7222	}
7223	}
7224
7225
7226
7227	/**
7228	* Runs the guest code using VT-x the normal way.
7229	*
7230	* @returns VBox status code.
7231	* @param pVM Pointer to the VM.
7232	* @param pVCpu Pointer to the VMCPU.
7233	* @param pCtx Pointer to the guest-CPU context.
7234	*
7235	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
7236	* @remarks Called with preemption disabled.
7237	*/
7238	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7239	{
7240	VMXTRANSIENT VmxTransient;
7241	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7242	int rc = VERR_INTERNAL_ERROR_5;
7243	uint32_t cLoops = 0;
7244
7245	for (;; cLoops++)
7246	{
7247	Assert(!HMR0SuspendPending());
7248	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7249	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7250	(unsigned)RTMpCpuId(), cLoops));
7251
7252	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7253	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7254	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7255	if (rc != VINF_SUCCESS)
7256	break;
7257
7258	/*
7259	* No longjmps to ring-3 from this point on!!!
7260	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7261	* This also disables flushing of the R0-logger instance (if any).
7262	*/
7263	VMMRZCallRing3Disable(pVCpu);
7264	VMMRZCallRing3RemoveNotification(pVCpu);
7265	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7266
7267	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7268	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7269
7270	/*
7271	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7272	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7273	*/
7274	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7275	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7276	{
7277	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7278	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7279	return rc;
7280	}
7281
7282	/* Handle the VM-exit. */
7283	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7284	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7285	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7286	HMVMX_START_EXIT_DISPATCH_PROF();
7287	#ifdef HMVMX_USE_FUNCTION_TABLE
7288	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7289	#else
7290	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7291	#endif
7292	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7293	if (rc != VINF_SUCCESS)
7294	break;
7295	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7296	{
7297	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7298	rc = VINF_EM_RAW_INTERRUPT;
7299	break;
7300	}
7301	}
7302
7303	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7304	return rc;
7305	}
7306
7307
7308	/**
7309	* Single steps guest code using VT-x.
7310	*
7311	* @returns VBox status code.
7312	* @param pVM Pointer to the VM.
7313	* @param pVCpu Pointer to the VMCPU.
7314	* @param pCtx Pointer to the guest-CPU context.
7315	*
7316	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7317	* @remarks Called with preemption disabled.
7318	*/
7319	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7320	{
7321	VMXTRANSIENT VmxTransient;
7322	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7323	int rc = VERR_INTERNAL_ERROR_5;
7324	uint32_t cLoops = 0;
7325	uint16_t uCsStart = pCtx->cs.Sel;
7326	uint64_t uRipStart = pCtx->rip;
7327
7328	for (;; cLoops++)
7329	{
7330	Assert(!HMR0SuspendPending());
7331	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7332	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7333	(unsigned)RTMpCpuId(), cLoops));
7334
7335	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7336	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7337	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7338	if (rc != VINF_SUCCESS)
7339	break;
7340
7341	/*
7342	* No longjmps to ring-3 from this point on!!!
7343	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7344	* This also disables flushing of the R0-logger instance (if any).
7345	*/
7346	VMMRZCallRing3Disable(pVCpu);
7347	VMMRZCallRing3RemoveNotification(pVCpu);
7348	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7349
7350	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7351	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7352
7353	/*
7354	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7355	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7356	*/
7357	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7358	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7359	{
7360	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7361	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7362	return rc;
7363	}
7364
7365	/* Handle the VM-exit. */
7366	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7367	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7368	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7369	HMVMX_START_EXIT_DISPATCH_PROF();
7370	#ifdef HMVMX_USE_FUNCTION_TABLE
7371	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7372	#else
7373	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7374	#endif
7375	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7376	if (rc != VINF_SUCCESS)
7377	break;
7378	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7379	{
7380	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7381	rc = VINF_EM_RAW_INTERRUPT;
7382	break;
7383	}
7384
7385	/*
7386	* Did the RIP change, if so, consider it a single step.
7387	* Otherwise, make sure one of the TFs gets set.
7388	*/
7389	int rc2 = hmR0VmxLoadGuestRip(pVCpu, pCtx);
7390	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7391	AssertRCReturn(rc2, rc2);
7392	if ( pCtx->rip != uRipStart
7393	\|\| pCtx->cs.Sel != uCsStart)
7394	{
7395	rc = VINF_EM_DBG_STEPPED;
7396	break;
7397	}
7398	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7399	}
7400
7401	/*
7402	* Clear the X86_EFL_TF if necessary .
7403	*/
7404	if (pVCpu->hm.s.fClearTrapFlag)
7405	{
7406	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
7407	AssertRCReturn(rc2, rc2);
7408	pVCpu->hm.s.fClearTrapFlag = false;
7409	pCtx->eflags.Bits.u1TF = 0;
7410	}
7411	/** @todo there seems to be issues with the resume flag when the monitor trap
7412	* flag is pending without being used. Seen early in bios init when
7413	* accessing APIC page in prot mode. */
7414
7415	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7416	return rc;
7417	}
7418
7419
7420	/**
7421	* Runs the guest code using VT-x.
7422	*
7423	* @returns VBox status code.
7424	* @param pVM Pointer to the VM.
7425	* @param pVCpu Pointer to the VMCPU.
7426	* @param pCtx Pointer to the guest-CPU context.
7427	*
7428	* @remarks Called with preemption disabled.
7429	*/
7430	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7431	{
7432	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7433	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7434
7435	int rc;
7436	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7437	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7438	else
7439	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7440
7441	if (rc == VERR_EM_INTERPRETER)
7442	rc = VINF_EM_RAW_EMULATE_INSTR;
7443	else if (rc == VINF_EM_RESET)
7444	rc = VINF_EM_TRIPLE_FAULT;
7445	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7446	return rc;
7447	}
7448
7449
7450	#ifndef HMVMX_USE_FUNCTION_TABLE
7451	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7452	{
7453	int rc;
7454	switch (rcReason)
7455	{
7456	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7457	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7458	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7459	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7460	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7461	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7462	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7463	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7464	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7465	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7466	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7467	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7468	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7469	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7470	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7471	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7472	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7473	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7474	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7475	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7476	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7477	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7478	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7479	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7480	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7481	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7482	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7483	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7484	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7485	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7486	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7487	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7488	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7489
7490	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7491	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7492	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7493	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7494	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7495	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7496	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7497	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7498	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7499
7500	case VMX_EXIT_VMCALL:
7501	case VMX_EXIT_VMCLEAR:
7502	case VMX_EXIT_VMLAUNCH:
7503	case VMX_EXIT_VMPTRLD:
7504	case VMX_EXIT_VMPTRST:
7505	case VMX_EXIT_VMREAD:
7506	case VMX_EXIT_VMRESUME:
7507	case VMX_EXIT_VMWRITE:
7508	case VMX_EXIT_VMXOFF:
7509	case VMX_EXIT_VMXON:
7510	case VMX_EXIT_INVEPT:
7511	case VMX_EXIT_INVVPID:
7512	case VMX_EXIT_VMFUNC:
7513	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7514	break;
7515	default:
7516	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7517	break;
7518	}
7519	return rc;
7520	}
7521	#endif
7522
7523	#ifdef DEBUG
7524	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7525	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7526	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7527
7528	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7529	do \
7530	{ \
7531	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7532	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7533	} while (0)
7534
7535	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7536	do { \
7537	AssertPtr(pVCpu); \
7538	AssertPtr(pMixedCtx); \
7539	AssertPtr(pVmxTransient); \
7540	Assert(pVmxTransient->fVMEntryFailed == false); \
7541	Assert(ASMIntAreEnabled()); \
7542	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7543	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7544	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7545	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7546	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7547	HMVMX_ASSERT_PREEMPT_CPUID(); \
7548	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7549	} while (0)
7550
7551	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7552	do { \
7553	Log4Func(("\n")); \
7554	} while(0)
7555	#else /* Release builds */
7556	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7557	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7558	#endif
7559
7560
7561	/**
7562	* Advances the guest RIP after reading it from the VMCS.
7563	*
7564	* @returns VBox status code.
7565	* @param pVCpu Pointer to the VMCPU.
7566	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7567	* out-of-sync. Make sure to update the required fields
7568	* before using them.
7569	* @param pVmxTransient Pointer to the VMX transient structure.
7570	*
7571	* @remarks No-long-jump zone!!!
7572	*/
7573	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7574	{
7575	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7576	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7577	AssertRCReturn(rc, rc);
7578
7579	pMixedCtx->rip += pVmxTransient->cbInstr;
7580	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7581	return rc;
7582	}
7583
7584
7585	/**
7586	* Tries to determine what part of the guest-state VT-x has deemed as invalid
7587	* and update error record fields accordingly.
7588	*
7589	* @return VMX_IGS_* return codes.
7590	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7591	* wrong with the guest state.
7592	*
7593	* @param pVM Pointer to the VM.
7594	* @param pVCpu Pointer to the VMCPU.
7595	* @param pCtx Pointer to the guest-CPU state.
7596	*/
7597	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7598	{
7599	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7600	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7601	uError = (err); \
7602	break; \
7603	} else do {} while (0)
7604	/* Duplicate of IEM_IS_CANONICAL(). */
7605	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7606
7607	int rc;
7608	uint64_t u64Val;
7609	uint32_t u32Val;
7610	uint32_t uError = VMX_IGS_ERROR;
7611	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7612
7613	do
7614	{
7615	/*
7616	* CR0.
7617	*/
7618	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7619	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7620	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7621	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7622	if (fUnrestrictedGuest)
7623	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
7624
7625	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7626	AssertRCBreak(rc);
7627	HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
7628	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
7629	if ( !fUnrestrictedGuest
7630	&& (u32Val & X86_CR0_PG)
7631	&& !(u32Val & X86_CR0_PE))
7632	{
7633	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7634	}
7635
7636	/*
7637	* CR4.
7638	*/
7639	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7640	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7641	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7642	AssertRCBreak(rc);
7643	HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
7644	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
7645
7646	/*
7647	* IA32_DEBUGCTL MSR.
7648	*/
7649	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7650	AssertRCBreak(rc);
7651	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7652	&& (u64Val & 0xfffffe3c)) /* Bits 31-9, bits 2-5 MBZ. */
7653	{
7654	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7655	}
7656	uint64_t u64DebugCtlMsr = u64Val;
7657
7658	#ifdef VBOX_STRICT
7659	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
7660	AssertRCBreak(rc);
7661	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
7662	#endif
7663	const bool fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
7664
7665	/*
7666	* RIP and RFLAGS.
7667	*/
7668	uint32_t u32EFlags;
7669	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7670	if (HMVMX_IS_64BIT_HOST_MODE())
7671	{
7672	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
7673	AssertRCBreak(rc);
7674	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
7675	if ( !fLongModeGuest
7676	\|\| !pCtx->cs.Attr.n.u1Long)
7677	{
7678	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
7679	}
7680	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
7681	* must be identical if the "IA32e mode guest" VM-entry control is 1
7682	* and CS.L is 1. No check applies if the CPU supports 64
7683	* linear-address bits. */
7684
7685	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
7686	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7687	AssertRCBreak(rc);
7688	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
7689	VMX_IGS_RFLAGS_RESERVED);
7690	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7691	u32EFlags = u64Val;
7692	}
7693	else
7694	#endif
7695	{
7696	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32EFlags);
7697	AssertRCBreak(rc);
7698	HMVMX_CHECK_BREAK(!(u32EFlags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
7699	HMVMX_CHECK_BREAK((u32EFlags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7700	}
7701
7702	if ( fLongModeGuest
7703	\|\| !(pCtx->cr0 & X86_CR0_PE))
7704	{
7705	HMVMX_CHECK_BREAK(!(u32EFlags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
7706	}
7707
7708	uint32_t u32EntryInfo;
7709	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7710	AssertRCBreak(rc);
7711	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
7712	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7713	{
7714	HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
7715	}
7716
7717	/*
7718	* 64-bit checks.
7719	*/
7720	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7721	if (HMVMX_IS_64BIT_HOST_MODE())
7722	{
7723	if ( fLongModeGuest
7724	&& !fUnrestrictedGuest)
7725	{
7726	HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
7727	HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
7728	}
7729
7730	if ( !fLongModeGuest
7731	&& (pCtx->cr4 & X86_CR4_PCIDE))
7732	{
7733	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
7734	}
7735
7736	/** @todo CR3 field must be such that bits 63:52 and bits in the range
7737	* 51:32 beyond the processor's physical-address width are 0. */
7738
7739	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7740	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
7741	{
7742	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
7743	}
7744
7745	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
7746	AssertRCBreak(rc);
7747	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
7748
7749	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
7750	AssertRCBreak(rc);
7751	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
7752	}
7753	#endif
7754
7755	/*
7756	* PERF_GLOBAL MSR.
7757	*/
7758	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
7759	{
7760	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
7761	AssertRCBreak(rc);
7762	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
7763	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63-35, bits 31-2 MBZ. */
7764	}
7765
7766	/*
7767	* PAT MSR.
7768	*/
7769	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
7770	{
7771	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
7772	AssertRCBreak(rc);
7773	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
7774	for (unsigned i = 0; i < 8; i++)
7775	{
7776	uint8_t u8Val = (u64Val & 0x7);
7777	if ( u8Val != 0 /* UC */
7778	\|\| u8Val != 1 /* WC */
7779	\|\| u8Val != 4 /* WT */
7780	\|\| u8Val != 5 /* WP */
7781	\|\| u8Val != 6 /* WB */
7782	\|\| u8Val != 7 /* UC- */)
7783	{
7784	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
7785	}
7786	u64Val >>= 3;
7787	}
7788	}
7789
7790	/*
7791	* EFER MSR.
7792	*/
7793	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
7794	{
7795	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
7796	AssertRCBreak(rc);
7797	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
7798	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63-12, bit 9, bits 7-1 MBZ. */
7799	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
7800	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
7801	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7802	\|\| (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
7803	}
7804
7805	/*
7806	* Segment registers.
7807	*/
7808	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7809	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
7810	if (!(u32EFlags & X86_EFL_VM))
7811	{
7812	/* CS */
7813	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
7814	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
7815	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
7816	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
7817	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7818	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
7819	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7820	/* CS cannot be loaded with NULL in protected mode. */
7821	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
7822	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
7823	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
7824	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
7825	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
7826	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
7827	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
7828	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
7829	else
7830	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
7831
7832	/* SS */
7833	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7834	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
7835	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
7836	if ( !(pCtx->cr0 & X86_CR0_PE)
7837	\|\| pCtx->cs.Attr.n.u4Type == 3)
7838	{
7839	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
7840	}
7841	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
7842	{
7843	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
7844	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
7845	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
7846	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
7847	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
7848	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7849	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
7850	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7851	}
7852
7853	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
7854	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
7855	{
7856	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
7857	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
7858	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7859	\|\| pCtx->ds.Attr.n.u4Type > 11
7860	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7861	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
7862	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
7863	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
7864	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7865	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
7866	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7867	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7868	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
7869	}
7870	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
7871	{
7872	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
7873	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
7874	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7875	\|\| pCtx->es.Attr.n.u4Type > 11
7876	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7877	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
7878	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
7879	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
7880	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7881	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
7882	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7883	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7884	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
7885	}
7886	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
7887	{
7888	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
7889	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
7890	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7891	\|\| pCtx->fs.Attr.n.u4Type > 11
7892	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
7893	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
7894	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
7895	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
7896	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7897	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
7898	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7899	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7900	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
7901	}
7902	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
7903	{
7904	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
7905	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
7906	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7907	\|\| pCtx->gs.Attr.n.u4Type > 11
7908	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
7909	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
7910	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
7911	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
7912	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
7913	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
7914	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
7915	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7916	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
7917	}
7918	/* 64-bit capable CPUs. */
7919	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7920	if (HMVMX_IS_64BIT_HOST_MODE())
7921	{
7922	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
7923	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
7924	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7925	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
7926	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
7927	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
7928	VMX_IGS_LONGMODE_SS_BASE_INVALID);
7929	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
7930	VMX_IGS_LONGMODE_DS_BASE_INVALID);
7931	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
7932	VMX_IGS_LONGMODE_ES_BASE_INVALID);
7933	}
7934	#endif
7935	}
7936	else
7937	{
7938	/* V86 mode checks. */
7939	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
7940	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7941	{
7942	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
7943	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
7944	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
7945	}
7946	else
7947	{
7948	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
7949	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
7950	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
7951	}
7952
7953	/* CS */
7954	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
7955	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
7956	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
7957	/* SS */
7958	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
7959	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
7960	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
7961	/* DS */
7962	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
7963	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
7964	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
7965	/* ES */
7966	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
7967	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
7968	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
7969	/* FS */
7970	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
7971	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
7972	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
7973	/* GS */
7974	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
7975	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
7976	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
7977	/* 64-bit capable CPUs. */
7978	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7979	if (HMVMX_IS_64BIT_HOST_MODE())
7980	{
7981	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
7982	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
7983	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7984	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
7985	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
7986	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
7987	VMX_IGS_LONGMODE_SS_BASE_INVALID);
7988	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
7989	VMX_IGS_LONGMODE_DS_BASE_INVALID);
7990	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
7991	VMX_IGS_LONGMODE_ES_BASE_INVALID);
7992	}
7993	#endif
7994	}
7995
7996	/*
7997	* TR.
7998	*/
7999	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8000	/* 64-bit capable CPUs. */
8001	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8002	if (HMVMX_IS_64BIT_HOST_MODE())
8003	{
8004	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8005	}
8006	#endif
8007	if (fLongModeGuest)
8008	{
8009	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8010	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8011	}
8012	else
8013	{
8014	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8015	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8016	VMX_IGS_TR_ATTR_TYPE_INVALID);
8017	}
8018	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8019	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8020	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11-8 MBZ. */
8021	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8022	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8023	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8024	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8025	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8026
8027	/*
8028	* GDTR and IDTR.
8029	*/
8030	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8031	if (HMVMX_IS_64BIT_HOST_MODE())
8032	{
8033	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8034	AssertRCBreak(rc);
8035	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8036
8037	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8038	AssertRCBreak(rc);
8039	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8040	}
8041	#endif
8042
8043	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8044	AssertRCBreak(rc);
8045	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8046
8047	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8048	AssertRCBreak(rc);
8049	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8050
8051	/*
8052	* Guest Non-Register State.
8053	*/
8054	/* Activity State. */
8055	uint32_t u32ActivityState;
8056	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8057	AssertRCBreak(rc);
8058	HMVMX_CHECK_BREAK( !u32ActivityState
8059	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.msr.vmx_misc)),
8060	VMX_IGS_ACTIVITY_STATE_INVALID);
8061	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8062	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8063	uint32_t u32IntrState;
8064	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8065	AssertRCBreak(rc);
8066	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8067	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8068	{
8069	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8070	}
8071
8072	/** @todo Activity state and injecting interrupts. Left as a todo since we
8073	* currently don't use activity states but ACTIVE. */
8074
8075	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8076	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8077
8078	/* Guest interruptibility-state. */
8079	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8080	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8081	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8082	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8083	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8084	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8085	HMVMX_CHECK_BREAK( (u32EFlags & X86_EFL_IF)
8086	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8087	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8088	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8089	{
8090	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8091	{
8092	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8093	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8094	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8095	}
8096	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8097	{
8098	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8099	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8100	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8101	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8102	}
8103	}
8104	/** @todo Assumes the processor is not in SMM. */
8105	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8106	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8107	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8108	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8109	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8110	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8111	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8112	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8113	{
8114	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8115	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8116	}
8117
8118	/* Pending debug exceptions. */
8119	if (HMVMX_IS_64BIT_HOST_MODE())
8120	{
8121	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8122	AssertRCBreak(rc);
8123	/* Bits 63-15, Bit 13, Bits 11-4 MBZ. */
8124	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8125	u32Val = u64Val; /* For pending debug exceptions checks below. */
8126	}
8127	else
8128	{
8129	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8130	AssertRCBreak(rc);
8131	/* Bits 31-15, Bit 13, Bits 11-4 MBZ. */
8132	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8133	}
8134
8135	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8136	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8137	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8138	{
8139	if ( (u32EFlags & X86_EFL_TF)
8140	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8141	{
8142	/* Bit 14 is PendingDebug.BS. */
8143	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8144	}
8145	if ( !(u32EFlags & X86_EFL_TF)
8146	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8147	{
8148	/* Bit 14 is PendingDebug.BS. */
8149	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8150	}
8151	}
8152
8153	/* VMCS link pointer. */
8154	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8155	AssertRCBreak(rc);
8156	if (u64Val != UINT64_C(0xffffffffffffffff))
8157	{
8158	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8159	/** @todo Bits beyond the processor's physical-address width MBZ. */
8160	/** @todo 32-bit located in memory referenced by value of this field (as a
8161	* physical address) must contain the processor's VMCS revision ID. */
8162	/** @todo SMM checks. */
8163	}
8164
8165	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8166
8167	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8168	if (uError == VMX_IGS_ERROR)
8169	uError = VMX_IGS_REASON_NOT_FOUND;
8170	} while (0);
8171
8172	pVCpu->hm.s.u32HMError = uError;
8173	return uError;
8174
8175	#undef HMVMX_ERROR_BREAK
8176	#undef HMVMX_CHECK_BREAK
8177	#undef HMVMX_IS_CANONICAL
8178	}
8179
8180	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8181	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8182	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8183
8184	/** @name VM-exit handlers.
8185	* @{
8186	*/
8187
8188	/**
8189	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8190	*/
8191	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8192	{
8193	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8194	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8195	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8196	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
8197	Assert(ASMIntAreEnabled());
8198	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8199	return VINF_SUCCESS;
8200	#endif
8201	return VINF_EM_RAW_INTERRUPT;
8202	}
8203
8204
8205	/**
8206	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8207	*/
8208	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8209	{
8210	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8211	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8212
8213	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8214	AssertRCReturn(rc, rc);
8215
8216	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8217	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8218	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8219	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8220
8221	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8222	{
8223	/*
8224	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8225	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8226	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8227	*
8228	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8229	*/
8230	VMXDispatchHostNmi();
8231	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
8232	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8233	return VINF_SUCCESS;
8234	}
8235
8236	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8237	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8238	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8239	{
8240	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8241	return VINF_SUCCESS;
8242	}
8243	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8244	{
8245	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8246	return rc;
8247	}
8248
8249	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8250	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8251	switch (uIntrType)
8252	{
8253	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8254	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8255	/* no break */
8256	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8257	{
8258	switch (uVector)
8259	{
8260	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8261	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8262	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8263	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8264	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8265	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8266	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8267	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8268	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8269	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8270	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8271	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8272	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8273	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8274	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8275	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8276	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8277	#endif
8278	default:
8279	{
8280	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8281	AssertRCReturn(rc, rc);
8282
8283	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8284	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8285	{
8286	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8287	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8288	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8289	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8290	AssertRCReturn(rc, rc);
8291	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8292	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8293	0 /* GCPtrFaultAddress */);
8294	AssertRCReturn(rc, rc);
8295	}
8296	else
8297	{
8298	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8299	pVCpu->hm.s.u32HMError = uVector;
8300	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8301	}
8302	break;
8303	}
8304	}
8305	break;
8306	}
8307
8308	default:
8309	{
8310	pVCpu->hm.s.u32HMError = uExitIntrInfo;
8311	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8312	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8313	break;
8314	}
8315	}
8316	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8317	return rc;
8318	}
8319
8320
8321	/**
8322	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8323	*/
8324	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8325	{
8326	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8327
8328	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8329	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8330	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8331	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8332	AssertRCReturn(rc, rc);
8333
8334	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8335	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8336	return VINF_SUCCESS;
8337	}
8338
8339
8340	/**
8341	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8342	*/
8343	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8344	{
8345	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8346	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8347	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8348	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8349	}
8350
8351
8352	/**
8353	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8354	*/
8355	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8356	{
8357	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8358	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8359	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8360	}
8361
8362
8363	/**
8364	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8365	*/
8366	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8367	{
8368	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8369	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8370	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8371	}
8372
8373
8374	/**
8375	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8376	*/
8377	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8378	{
8379	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8380	PVM pVM = pVCpu->CTX_SUFF(pVM);
8381	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8382	if (RT_LIKELY(rc == VINF_SUCCESS))
8383	{
8384	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8385	Assert(pVmxTransient->cbInstr == 2);
8386	}
8387	else
8388	{
8389	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8390	rc = VERR_EM_INTERPRETER;
8391	}
8392	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8393	return rc;
8394	}
8395
8396
8397	/**
8398	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8399	*/
8400	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8401	{
8402	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8403	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8404	AssertRCReturn(rc, rc);
8405
8406	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8407	return VINF_EM_RAW_EMULATE_INSTR;
8408
8409	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8410	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8411	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8412	}
8413
8414
8415	/**
8416	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8417	*/
8418	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8419	{
8420	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8421	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8422	AssertRCReturn(rc, rc);
8423
8424	PVM pVM = pVCpu->CTX_SUFF(pVM);
8425	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8426	if (RT_LIKELY(rc == VINF_SUCCESS))
8427	{
8428	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8429	Assert(pVmxTransient->cbInstr == 2);
8430	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8431	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8432	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8433	}
8434	else
8435	{
8436	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8437	rc = VERR_EM_INTERPRETER;
8438	}
8439	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8440	return rc;
8441	}
8442
8443
8444	/**
8445	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8446	*/
8447	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8448	{
8449	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8450	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8451	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8452	AssertRCReturn(rc, rc);
8453
8454	PVM pVM = pVCpu->CTX_SUFF(pVM);
8455	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8456	if (RT_LIKELY(rc == VINF_SUCCESS))
8457	{
8458	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8459	Assert(pVmxTransient->cbInstr == 3);
8460	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8461	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8462	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8463	}
8464	else
8465	{
8466	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8467	rc = VERR_EM_INTERPRETER;
8468	}
8469	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8470	return rc;
8471	}
8472
8473
8474	/**
8475	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8476	*/
8477	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8478	{
8479	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8480	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8481	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8482	AssertRCReturn(rc, rc);
8483
8484	PVM pVM = pVCpu->CTX_SUFF(pVM);
8485	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8486	if (RT_LIKELY(rc == VINF_SUCCESS))
8487	{
8488	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8489	Assert(pVmxTransient->cbInstr == 2);
8490	}
8491	else
8492	{
8493	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8494	rc = VERR_EM_INTERPRETER;
8495	}
8496	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8497	return rc;
8498	}
8499
8500
8501	/**
8502	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8503	*/
8504	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8505	{
8506	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8507	PVM pVM = pVCpu->CTX_SUFF(pVM);
8508	Assert(!pVM->hm.s.fNestedPaging);
8509
8510	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8511	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8512	AssertRCReturn(rc, rc);
8513
8514	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8515	rc = VBOXSTRICTRC_VAL(rc2);
8516	if (RT_LIKELY(rc == VINF_SUCCESS))
8517	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8518	else
8519	{
8520	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8521	pVmxTransient->uExitQualification, rc));
8522	}
8523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8524	return rc;
8525	}
8526
8527
8528	/**
8529	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8530	*/
8531	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8532	{
8533	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8534	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8535	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8536	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8537	AssertRCReturn(rc, rc);
8538
8539	PVM pVM = pVCpu->CTX_SUFF(pVM);
8540	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8541	if (RT_LIKELY(rc == VINF_SUCCESS))
8542	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8543	else
8544	{
8545	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8546	rc = VERR_EM_INTERPRETER;
8547	}
8548	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8549	return rc;
8550	}
8551
8552
8553	/**
8554	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8555	*/
8556	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8557	{
8558	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8559	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8560	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8561	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8562	AssertRCReturn(rc, rc);
8563
8564	PVM pVM = pVCpu->CTX_SUFF(pVM);
8565	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8566	rc = VBOXSTRICTRC_VAL(rc2);
8567	if (RT_LIKELY( rc == VINF_SUCCESS
8568	\|\| rc == VINF_EM_HALT))
8569	{
8570	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8571	AssertRCReturn(rc3, rc3);
8572
8573	if ( rc == VINF_EM_HALT
8574	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8575	{
8576	rc = VINF_SUCCESS;
8577	}
8578	}
8579	else
8580	{
8581	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8582	rc = VERR_EM_INTERPRETER;
8583	}
8584	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
8585	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8586	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8587	return rc;
8588	}
8589
8590
8591	/**
8592	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8593	*/
8594	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8595	{
8596	/*
8597	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8598	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8599	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
8600	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8601	*/
8602	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8603	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8604	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8605	}
8606
8607
8608	/**
8609	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8610	*/
8611	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8612	{
8613	/*
8614	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8615	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8616	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
8617	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8618	*/
8619	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8620	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8621	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8622	}
8623
8624
8625	/**
8626	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
8627	*/
8628	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8629	{
8630	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
8631	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8632	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
8633	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8634	}
8635
8636
8637	/**
8638	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
8639	*/
8640	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8641	{
8642	/*
8643	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
8644	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
8645	* See Intel spec. 25.3 "Other Causes of VM-exits".
8646	*/
8647	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8648	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
8649	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8650	}
8651
8652
8653	/**
8654	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
8655	* VM-exit.
8656	*/
8657	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8658	{
8659	/*
8660	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
8661	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
8662	*
8663	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
8664	* See Intel spec. "23.8 Restrictions on VMX operation".
8665	*/
8666	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8667	return VINF_SUCCESS;
8668	}
8669
8670
8671	/**
8672	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
8673	* VM-exit.
8674	*/
8675	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8676	{
8677	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8678	return VINF_EM_RESET;
8679	}
8680
8681
8682	/**
8683	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
8684	*/
8685	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8686	{
8687	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8688	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
8689	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8690	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8691	AssertRCReturn(rc, rc);
8692
8693	pMixedCtx->rip++;
8694	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8695	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
8696	rc = VINF_SUCCESS;
8697	else
8698	rc = VINF_EM_HALT;
8699
8700	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8701	return rc;
8702	}
8703
8704
8705	/**
8706	* VM-exit handler for instructions that result in a #UD exception delivered to
8707	* the guest.
8708	*/
8709	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8710	{
8711	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8712	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
8713	return VINF_SUCCESS;
8714	}
8715
8716
8717	/**
8718	* VM-exit handler for expiry of the VMX preemption timer.
8719	*/
8720	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8721	{
8722	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8723
8724	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
8725	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8726
8727	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
8728	PVM pVM = pVCpu->CTX_SUFF(pVM);
8729	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
8730	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
8731	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
8732	}
8733
8734
8735	/**
8736	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
8737	*/
8738	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8739	{
8740	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8741
8742	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
8743	/** @todo check if XSETBV is supported by the recompiler. */
8744	return VERR_EM_INTERPRETER;
8745	}
8746
8747
8748	/**
8749	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
8750	*/
8751	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8752	{
8753	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8754
8755	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
8756	/** @todo implement EMInterpretInvpcid() */
8757	return VERR_EM_INTERPRETER;
8758	}
8759
8760
8761	/**
8762	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
8763	* Error VM-exit.
8764	*/
8765	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8766	{
8767	uint32_t uIntrState;
8768	HMVMXHCUINTREG uHCReg;
8769	uint64_t u64Val;
8770	uint32_t u32Val;
8771
8772	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
8773	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
8774	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
8775	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
8776	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8777	AssertRCReturn(rc, rc);
8778
8779	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
8780	NOREF(uInvalidReason);
8781
8782	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
8783	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
8784	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
8785	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
8786
8787	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
8788	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
8789	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
8790	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
8791	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
8792	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
8793	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
8794	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
8795	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
8796	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
8797	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
8798	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
8799
8800	PVM pVM = pVCpu->CTX_SUFF(pVM);
8801	HMDumpRegs(pVM, pVCpu, pMixedCtx);
8802
8803	return VERR_VMX_INVALID_GUEST_STATE;
8804	}
8805
8806
8807	/**
8808	* VM-exit handler for VM-entry failure due to an MSR-load
8809	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
8810	*/
8811	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8812	{
8813	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8814	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8815	}
8816
8817
8818	/**
8819	* VM-exit handler for VM-entry failure due to a machine-check event
8820	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
8821	*/
8822	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8823	{
8824	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8825	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8826	}
8827
8828
8829	/**
8830	* VM-exit handler for all undefined reasons. Should never ever happen.. in
8831	* theory.
8832	*/
8833	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8834	{
8835	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
8836	return VERR_VMX_UNDEFINED_EXIT_CODE;
8837	}
8838
8839
8840	/**
8841	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
8842	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
8843	* Conditional VM-exit.
8844	*/
8845	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8846	{
8847	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8848
8849	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
8850	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
8851	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
8852	return VERR_EM_INTERPRETER;
8853	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8854	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8855	}
8856
8857
8858	/**
8859	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
8860	*/
8861	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8862	{
8863	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8864
8865	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
8866	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
8867	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
8868	return VERR_EM_INTERPRETER;
8869	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8870	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8871	}
8872
8873
8874	/**
8875	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
8876	*/
8877	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8878	{
8879	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8880
8881	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
8882	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8883	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8884	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8885	AssertRCReturn(rc, rc);
8886
8887	PVM pVM = pVCpu->CTX_SUFF(pVM);
8888	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8889	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
8890	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
8891	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
8892
8893	if (RT_LIKELY(rc == VINF_SUCCESS))
8894	{
8895	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8896	Assert(pVmxTransient->cbInstr == 2);
8897	}
8898	return rc;
8899	}
8900
8901
8902	/**
8903	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
8904	*/
8905	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8906	{
8907	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8908	PVM pVM = pVCpu->CTX_SUFF(pVM);
8909	int rc = VINF_SUCCESS;
8910
8911	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
8912	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8913	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8914	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8915	AssertRCReturn(rc, rc);
8916	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
8917
8918	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8919	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
8920	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
8921
8922	if (RT_LIKELY(rc == VINF_SUCCESS))
8923	{
8924	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8925
8926	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
8927	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
8928	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
8929	{
8930	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
8931	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
8932	EMInterpretWrmsr() changes it. */
8933	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8934	}
8935	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
8936	{
8937	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8938	AssertRCReturn(rc, rc);
8939	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
8940	}
8941	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
8942	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8943
8944	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
8945	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8946	{
8947	switch (pMixedCtx->ecx)
8948	{
8949	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
8950	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
8951	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
8952	case MSR_K8_FS_BASE: /* no break */
8953	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
8954	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
8955	}
8956	}
8957	#ifdef VBOX_STRICT
8958	else
8959	{
8960	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
8961	switch (pMixedCtx->ecx)
8962	{
8963	case MSR_IA32_SYSENTER_CS:
8964	case MSR_IA32_SYSENTER_EIP:
8965	case MSR_IA32_SYSENTER_ESP:
8966	case MSR_K8_FS_BASE:
8967	case MSR_K8_GS_BASE:
8968	{
8969	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
8970	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8971	}
8972
8973	case MSR_K8_LSTAR:
8974	case MSR_K6_STAR:
8975	case MSR_K8_SF_MASK:
8976	case MSR_K8_TSC_AUX:
8977	case MSR_K8_KERNEL_GS_BASE:
8978	{
8979	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
8980	pMixedCtx->ecx));
8981	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8982	}
8983	}
8984	}
8985	#endif /* VBOX_STRICT */
8986	}
8987	return rc;
8988	}
8989
8990
8991	/**
8992	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8993	*/
8994	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8995	{
8996	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8997
8998	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
8999	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9000	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9001	return VERR_EM_INTERPRETER;
9002	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9003	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9004	}
9005
9006
9007	/**
9008	* VM-exit handler for when the TPR value is lowered below the specified
9009	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9010	*/
9011	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9012	{
9013	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9014	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9015
9016	/*
9017	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9018	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9019	* resume guest execution.
9020	*/
9021	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9022	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9023	return VINF_SUCCESS;
9024	}
9025
9026
9027	/**
9028	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9029	* VM-exit.
9030	*
9031	* @retval VINF_SUCCESS when guest execution can continue.
9032	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9033	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9034	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9035	* recompiler.
9036	*/
9037	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9038	{
9039	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9040	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9041	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9042	AssertRCReturn(rc, rc);
9043
9044	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9045	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9046	PVM pVM = pVCpu->CTX_SUFF(pVM);
9047	switch (uAccessType)
9048	{
9049	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9050	{
9051	#if 0
9052	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9053	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9054	#else
9055	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9056	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9057	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9058	#endif
9059	AssertRCReturn(rc, rc);
9060
9061	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9062	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9063	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9064	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9065
9066	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9067	{
9068	case 0: /* CR0 */
9069	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9070	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9071	break;
9072	case 2: /* C2 **/
9073	/* Nothing to do here, CR2 it's not part of the VMCS. */
9074	break;
9075	case 3: /* CR3 */
9076	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9077	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9078	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
9079	break;
9080	case 4: /* CR4 */
9081	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9082	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
9083	break;
9084	case 8: /* CR8 */
9085	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9086	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9087	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9088	break;
9089	default:
9090	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9091	break;
9092	}
9093
9094	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9095	break;
9096	}
9097
9098	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9099	{
9100	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9101	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9102	AssertRCReturn(rc, rc);
9103	Assert( !pVM->hm.s.fNestedPaging
9104	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9105	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9106
9107	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9108	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9109	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9110
9111	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9112	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9113	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9114	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9116	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9117	break;
9118	}
9119
9120	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9121	{
9122	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9123	AssertRCReturn(rc, rc);
9124	rc = EMInterpretCLTS(pVM, pVCpu);
9125	AssertRCReturn(rc, rc);
9126	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9127	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9128	Log4(("CRX CLTS write rc=%d\n", rc));
9129	break;
9130	}
9131
9132	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9133	{
9134	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9135	AssertRCReturn(rc, rc);
9136	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9137	if (RT_LIKELY(rc == VINF_SUCCESS))
9138	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9139	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9140	Log4(("CRX LMSW write rc=%d\n", rc));
9141	break;
9142	}
9143
9144	default:
9145	{
9146	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9147	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9148	}
9149	}
9150
9151	/* Validate possible error codes. */
9152	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9153	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9154	if (RT_SUCCESS(rc))
9155	{
9156	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9157	AssertRCReturn(rc2, rc2);
9158	}
9159
9160	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9161	return rc;
9162	}
9163
9164
9165	/**
9166	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9167	* VM-exit.
9168	*/
9169	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9170	{
9171	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9172	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9173
9174	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9175	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9176	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9177	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9178	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9179	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9180	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9181	AssertRCReturn(rc2, rc2);
9182
9183	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9184	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9185	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9186	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9187	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9188	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9189	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9190
9191	/* I/O operation lookup arrays. */
9192	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9193	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9194
9195	VBOXSTRICTRC rcStrict;
9196	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9197	const uint32_t cbInstr = pVmxTransient->cbInstr;
9198	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9199	PVM pVM = pVCpu->CTX_SUFF(pVM);
9200	if (fIOString)
9201	{
9202	/*
9203	* INS/OUTS - I/O String instruction.
9204	*
9205	* Use instruction-information if available, otherwise fall back on
9206	* interpreting the instruction.
9207	*/
9208	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9209	#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9210	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9211	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.msr.vmx_basic_info))
9212	{
9213	rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9214	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9215	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9216	AssertRCReturn(rc2, rc2);
9217	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9218	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9219	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9220	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9221	if (fIOWrite)
9222	{
9223	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9224	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9225	//if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9226	// hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9227	// pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9228	}
9229	else
9230	{
9231	AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9232	("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9233	VERR_HMVMX_IPE_4);
9234	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9235	//if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9236	// hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9237	}
9238	}
9239	else
9240	{
9241	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9242	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9243	AssertRCReturn(rc2, rc2);
9244	rcStrict = IEMExecOne(pVCpu);
9245	}
9246	/** @todo IEM needs to be setting these flags somehow. */
9247	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9248	fUpdateRipAlready = true;
9249
9250	#else
9251	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9252	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9253	if (RT_SUCCESS(rcStrict))
9254	{
9255	if (fIOWrite)
9256	{
9257	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9258	(DISCPUMODE)pDis->uAddrMode, cbValue);
9259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9260	}
9261	else
9262	{
9263	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9264	(DISCPUMODE)pDis->uAddrMode, cbValue);
9265	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9266	}
9267	}
9268	else
9269	{
9270	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9271	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9272	}
9273	#endif
9274	}
9275	else
9276	{
9277	/*
9278	* IN/OUT - I/O instruction.
9279	*/
9280	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9281	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9282	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9283	if (fIOWrite)
9284	{
9285	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9286	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9287	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9288	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9289	}
9290	else
9291	{
9292	uint32_t u32Result = 0;
9293	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9294	if (IOM_SUCCESS(rcStrict))
9295	{
9296	/* Save result of I/O IN instr. in AL/AX/EAX. */
9297	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9298	}
9299	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9300	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9302	}
9303	}
9304
9305	if (IOM_SUCCESS(rcStrict))
9306	{
9307	if (!fUpdateRipAlready)
9308	{
9309	pMixedCtx->rip += cbInstr;
9310	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9311	}
9312
9313	/*
9314	* If any I/O breakpoints are armed, we need to check if one triggered
9315	* and take appropriate action.
9316	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9317	*/
9318	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9319	AssertRCReturn(rc2, rc2);
9320
9321	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9322	* execution engines about whether hyper BPs and such are pending. */
9323	uint32_t const uDr7 = pMixedCtx->dr[7];
9324	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9325	&& X86_DR7_ANY_RW_IO(uDr7)
9326	&& (pMixedCtx->cr4 & X86_CR4_DE))
9327	\|\| DBGFBpIsHwIoArmed(pVM)))
9328	{
9329	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9330	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9331
9332	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9333	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9334	{
9335	/* Raise #DB. */
9336	if (fIsGuestDbgActive)
9337	ASMSetDR6(pMixedCtx->dr[6]);
9338	if (pMixedCtx->dr[7] != uDr7)
9339	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9340
9341	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9342	}
9343	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9344	else if ( rcStrict2 != VINF_SUCCESS
9345	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9346	rcStrict = rcStrict2;
9347	}
9348	}
9349
9350	#ifdef DEBUG
9351	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9352	Assert(!fIOWrite);
9353	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9354	Assert(fIOWrite);
9355	else
9356	{
9357	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9358	* statuses, that the VMM device and some others may return. See
9359	* IOM_SUCCESS() for guidance. */
9360	AssertMsg( RT_FAILURE(rcStrict)
9361	\|\| rcStrict == VINF_SUCCESS
9362	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9363	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9364	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9365	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9366	}
9367	#endif
9368
9369	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9370	return VBOXSTRICTRC_TODO(rcStrict);
9371	}
9372
9373
9374	/**
9375	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9376	* VM-exit.
9377	*/
9378	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9379	{
9380	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9381
9382	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9383	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9384	AssertRCReturn(rc, rc);
9385	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9386	{
9387	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9388	AssertRCReturn(rc, rc);
9389	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9390	{
9391	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9392
9393	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9394	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9395	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9396	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9397	{
9398	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9399	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9400
9401	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9402	Assert(!pVCpu->hm.s.Event.fPending);
9403	pVCpu->hm.s.Event.fPending = true;
9404	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9405	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9406	AssertRCReturn(rc, rc);
9407	if (fErrorCodeValid)
9408	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9409	else
9410	pVCpu->hm.s.Event.u32ErrCode = 0;
9411	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9412	&& uVector == X86_XCPT_PF)
9413	{
9414	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9415	}
9416
9417	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9418	}
9419	}
9420	}
9421
9422	/** @todo Emulate task switch someday, currently just going back to ring-3 for
9423	* emulation. */
9424	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9425	return VERR_EM_INTERPRETER;
9426	}
9427
9428
9429	/**
9430	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9431	*/
9432	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9433	{
9434	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9435	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9436	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9437	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9438	AssertRCReturn(rc, rc);
9439	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9440	return VINF_EM_DBG_STEPPED;
9441	}
9442
9443
9444	/**
9445	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9446	*/
9447	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9448	{
9449	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9450
9451	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9452	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9453	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9454	return VINF_SUCCESS;
9455	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9456	return rc;
9457
9458	#if 0
9459	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9460	* just sync the whole thing. */
9461	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9462	#else
9463	/* Aggressive state sync. for now. */
9464	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9465	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9466	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9467	#endif
9468	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9469	AssertRCReturn(rc, rc);
9470
9471	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9472	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9473	switch (uAccessType)
9474	{
9475	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9476	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9477	{
9478	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9479	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9480	{
9481	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9482	}
9483
9484	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9485	GCPhys &= PAGE_BASE_GC_MASK;
9486	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9487	PVM pVM = pVCpu->CTX_SUFF(pVM);
9488	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9489	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9490
9491	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9492	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9493	CPUMCTX2CORE(pMixedCtx), GCPhys);
9494	rc = VBOXSTRICTRC_VAL(rc2);
9495	Log4(("ApicAccess rc=%d\n", rc));
9496	if ( rc == VINF_SUCCESS
9497	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9498	\|\| rc == VERR_PAGE_NOT_PRESENT)
9499	{
9500	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9501	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9502	rc = VINF_SUCCESS;
9503	}
9504	break;
9505	}
9506
9507	default:
9508	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9509	rc = VINF_EM_RAW_EMULATE_INSTR;
9510	break;
9511	}
9512
9513	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9514	return rc;
9515	}
9516
9517
9518	/**
9519	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9520	* VM-exit.
9521	*/
9522	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9523	{
9524	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9525
9526	/* We should -not- get this VM-exit if the guest is debugging. */
9527	if (CPUMIsGuestDebugStateActive(pVCpu))
9528	{
9529	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9530	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9531	}
9532
9533	int rc = VERR_INTERNAL_ERROR_5;
9534	if ( !DBGFIsStepping(pVCpu)
9535	&& !pVCpu->hm.s.fSingleInstruction
9536	&& !CPUMIsHyperDebugStateActive(pVCpu))
9537	{
9538	/* Don't intercept MOV DRx and #DB any more. */
9539	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9540	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9541	AssertRCReturn(rc, rc);
9542
9543	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9544	{
9545	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9546	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9547	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9548	AssertRCReturn(rc, rc);
9549	#endif
9550	}
9551
9552	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9553	PVM pVM = pVCpu->CTX_SUFF(pVM);
9554	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9555	Assert(CPUMIsGuestDebugStateActive(pVCpu));
9556
9557	#ifdef VBOX_WITH_STATISTICS
9558	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9559	AssertRCReturn(rc, rc);
9560	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9561	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9562	else
9563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9564	#endif
9565	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9566	return VINF_SUCCESS;
9567	}
9568
9569	/*
9570	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9571	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9572	*/
9573	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9574	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9575	AssertRCReturn(rc, rc);
9576
9577	PVM pVM = pVCpu->CTX_SUFF(pVM);
9578	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9579	{
9580	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9581	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9582	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9583	if (RT_SUCCESS(rc))
9584	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9586	}
9587	else
9588	{
9589	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9590	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9591	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9593	}
9594
9595	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9596	if (RT_SUCCESS(rc))
9597	{
9598	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9599	AssertRCReturn(rc2, rc2);
9600	}
9601	return rc;
9602	}
9603
9604
9605	/**
9606	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9607	* Conditional VM-exit.
9608	*/
9609	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9610	{
9611	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9612	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9613
9614	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9615	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9616	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9617	return VINF_SUCCESS;
9618	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9619	return rc;
9620
9621	RTGCPHYS GCPhys = 0;
9622	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9623
9624	#if 0
9625	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9626	#else
9627	/* Aggressive state sync. for now. */
9628	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9629	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9630	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9631	#endif
9632	AssertRCReturn(rc, rc);
9633
9634	/*
9635	* If we succeed, resume guest execution.
9636	* If we fail in interpreting the instruction because we couldn't get the guest physical address
9637	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
9638	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
9639	* weird case. See @bugref{6043}.
9640	*/
9641	PVM pVM = pVCpu->CTX_SUFF(pVM);
9642	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
9643	rc = VBOXSTRICTRC_VAL(rc2);
9644	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
9645	if ( rc == VINF_SUCCESS
9646	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9647	\|\| rc == VERR_PAGE_NOT_PRESENT)
9648	{
9649	/* Successfully handled MMIO operation. */
9650	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9651	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9652	rc = VINF_SUCCESS;
9653	}
9654	return rc;
9655	}
9656
9657
9658	/**
9659	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
9660	* VM-exit.
9661	*/
9662	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9663	{
9664	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9665	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9666
9667	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9668	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9669	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9670	return VINF_SUCCESS;
9671	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9672	return rc;
9673
9674	RTGCPHYS GCPhys = 0;
9675	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9676	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9677	#if 0
9678	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9679	#else
9680	/* Aggressive state sync. for now. */
9681	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9682	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9683	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9684	#endif
9685	AssertRCReturn(rc, rc);
9686
9687	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
9688	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
9689
9690	RTGCUINT uErrorCode = 0;
9691	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
9692	uErrorCode \|= X86_TRAP_PF_ID;
9693	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
9694	uErrorCode \|= X86_TRAP_PF_RW;
9695	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
9696	uErrorCode \|= X86_TRAP_PF_P;
9697
9698	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
9699
9700	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
9701	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9702
9703	/* Handle the pagefault trap for the nested shadow table. */
9704	PVM pVM = pVCpu->CTX_SUFF(pVM);
9705	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
9706	TRPMResetTrap(pVCpu);
9707
9708	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
9709	if ( rc == VINF_SUCCESS
9710	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9711	\|\| rc == VERR_PAGE_NOT_PRESENT)
9712	{
9713	/* Successfully synced our nested page tables. */
9714	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
9715	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
9716	return VINF_SUCCESS;
9717	}
9718
9719	Log4(("EPT return to ring-3 rc=%d\n"));
9720	return rc;
9721	}
9722
9723	/** @} */
9724
9725	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9726	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
9727	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9728
9729	/** @name VM-exit exception handlers.
9730	* @{
9731	*/
9732
9733	/**
9734	* VM-exit exception handler for #MF (Math Fault: floating point exception).
9735	*/
9736	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9737	{
9738	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9739	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
9740
9741	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9742	AssertRCReturn(rc, rc);
9743
9744	if (!(pMixedCtx->cr0 & X86_CR0_NE))
9745	{
9746	/* Old-style FPU error reporting needs some extra work. */
9747	/** @todo don't fall back to the recompiler, but do it manually. */
9748	return VERR_EM_INTERPRETER;
9749	}
9750
9751	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9752	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9753	return rc;
9754	}
9755
9756
9757	/**
9758	* VM-exit exception handler for #BP (Breakpoint exception).
9759	*/
9760	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9761	{
9762	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9763	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
9764
9765	/** @todo Try optimize this by not saving the entire guest state unless
9766	* really needed. */
9767	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9768	AssertRCReturn(rc, rc);
9769
9770	PVM pVM = pVCpu->CTX_SUFF(pVM);
9771	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9772	if (rc == VINF_EM_RAW_GUEST_TRAP)
9773	{
9774	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9775	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9776	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9777	AssertRCReturn(rc, rc);
9778
9779	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9780	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9781	}
9782
9783	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
9784	return rc;
9785	}
9786
9787
9788	/**
9789	* VM-exit exception handler for #DB (Debug exception).
9790	*/
9791	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9792	{
9793	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
9795	Log6(("XcptDB\n"));
9796
9797	/*
9798	* Get the DR6-like values from the exit qualification and pass it to DBGF
9799	* for processing.
9800	*/
9801	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9802	AssertRCReturn(rc, rc);
9803
9804	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
9805	uint64_t uDR6 = X86_DR6_INIT_VAL;
9806	uDR6 \|= ( pVmxTransient->uExitQualification
9807	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
9808
9809	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
9810	if (rc == VINF_EM_RAW_GUEST_TRAP)
9811	{
9812	/*
9813	* The exception was for the guest. Update DR6, DR7.GD and
9814	* IA32_DEBUGCTL.LBR before forwarding it.
9815	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
9816	*/
9817	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
9818	pMixedCtx->dr[6] \|= uDR6;
9819	if (CPUMIsGuestDebugStateActive(pVCpu))
9820	ASMSetDR6(pMixedCtx->dr[6]);
9821
9822	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9823	AssertRCReturn(rc, rc);
9824
9825	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
9826	pMixedCtx->dr[7] &= ~X86_DR7_GD;
9827
9828	/* Paranoia. */
9829	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
9830	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
9831
9832	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
9833	AssertRCReturn(rc, rc);
9834
9835	/*
9836	* Raise #DB in the guest.
9837	*/
9838	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9839	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9840	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9841	AssertRCReturn(rc, rc);
9842	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9843	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9844	return VINF_SUCCESS;
9845	}
9846
9847	/*
9848	* Not a guest trap, must be a hypervisor related debug event then.
9849	* Update DR6 in case someone is interested in it.
9850	*/
9851	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
9852	AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
9853	CPUMSetHyperDR6(pVCpu, uDR6);
9854
9855	return rc;
9856	}
9857
9858
9859	/**
9860	* VM-exit exception handler for #NM (Device-not-available exception: floating
9861	* point exception).
9862	*/
9863	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9864	{
9865	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9866
9867	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9868	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
9869	#endif
9870
9871	/* We require CR0 and EFER. EFER is always up-to-date. */
9872	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9873	AssertRCReturn(rc, rc);
9874
9875	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
9876	PVM pVM = pVCpu->CTX_SUFF(pVM);
9877	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
9878	if (rc == VINF_SUCCESS)
9879	{
9880	Assert(CPUMIsGuestFPUStateActive(pVCpu));
9881	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9882	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
9883	return VINF_SUCCESS;
9884	}
9885
9886	/* Forward #NM to the guest. */
9887	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
9888	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9889	AssertRCReturn(rc, rc);
9890	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9891	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
9892	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
9893	return rc;
9894	}
9895
9896
9897	/**
9898	* VM-exit exception handler for #GP (General-protection exception).
9899	*
9900	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
9901	*/
9902	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9903	{
9904	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9905	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
9906
9907	int rc = VERR_INTERNAL_ERROR_5;
9908	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9909	{
9910	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9911	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
9912	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9913	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9914	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9915	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9916	AssertRCReturn(rc, rc);
9917	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
9918	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
9919	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9920	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9921	return rc;
9922	#else
9923	/* We don't intercept #GP. */
9924	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
9925	return VERR_VMX_UNEXPECTED_EXCEPTION;
9926	#endif
9927	}
9928
9929	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9930	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
9931
9932	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
9933	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9934	AssertRCReturn(rc, rc);
9935
9936	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9937	uint32_t cbOp = 0;
9938	PVM pVM = pVCpu->CTX_SUFF(pVM);
9939	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
9940	if (RT_SUCCESS(rc))
9941	{
9942	rc = VINF_SUCCESS;
9943	Assert(cbOp == pDis->cbInstr);
9944	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9945	switch (pDis->pCurInstr->uOpcode)
9946	{
9947	case OP_CLI:
9948	{
9949	pMixedCtx->eflags.Bits.u1IF = 0;
9950	pMixedCtx->rip += pDis->cbInstr;
9951	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9952	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
9953	break;
9954	}
9955
9956	case OP_STI:
9957	{
9958	pMixedCtx->eflags.Bits.u1IF = 1;
9959	pMixedCtx->rip += pDis->cbInstr;
9960	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
9961	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9962	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9963	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
9964	break;
9965	}
9966
9967	case OP_HLT:
9968	{
9969	rc = VINF_EM_HALT;
9970	pMixedCtx->rip += pDis->cbInstr;
9971	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9973	break;
9974	}
9975
9976	case OP_POPF:
9977	{
9978	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9979	uint32_t cbParm = 0;
9980	uint32_t uMask = 0;
9981	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9982	{
9983	cbParm = 4;
9984	uMask = 0xffffffff;
9985	}
9986	else
9987	{
9988	cbParm = 2;
9989	uMask = 0xffff;
9990	}
9991
9992	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
9993	RTGCPTR GCPtrStack = 0;
9994	X86EFLAGS uEflags;
9995	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9996	&GCPtrStack);
9997	if (RT_SUCCESS(rc))
9998	{
9999	Assert(sizeof(uEflags.u32) >= cbParm);
10000	uEflags.u32 = 0;
10001	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
10002	}
10003	if (RT_FAILURE(rc))
10004	{
10005	rc = VERR_EM_INTERPRETER;
10006	break;
10007	}
10008	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10009	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10010	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
10011	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10012	pMixedCtx->eflags.Bits.u1RF = 0;
10013	pMixedCtx->esp += cbParm;
10014	pMixedCtx->esp &= uMask;
10015	pMixedCtx->rip += pDis->cbInstr;
10016	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
10017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10018	break;
10019	}
10020
10021	case OP_PUSHF:
10022	{
10023	uint32_t cbParm = 0;
10024	uint32_t uMask = 0;
10025	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10026	{
10027	cbParm = 4;
10028	uMask = 0xffffffff;
10029	}
10030	else
10031	{
10032	cbParm = 2;
10033	uMask = 0xffff;
10034	}
10035
10036	/* Get the stack pointer & push the contents of eflags onto the stack. */
10037	RTGCPTR GCPtrStack = 0;
10038	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10039	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10040	if (RT_FAILURE(rc))
10041	{
10042	rc = VERR_EM_INTERPRETER;
10043	break;
10044	}
10045	X86EFLAGS uEflags;
10046	uEflags = pMixedCtx->eflags;
10047	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10048	uEflags.Bits.u1RF = 0;
10049	uEflags.Bits.u1VM = 0;
10050
10051	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
10052	if (RT_FAILURE(rc))
10053	{
10054	rc = VERR_EM_INTERPRETER;
10055	break;
10056	}
10057	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
10058	pMixedCtx->esp -= cbParm;
10059	pMixedCtx->esp &= uMask;
10060	pMixedCtx->rip += pDis->cbInstr;
10061	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
10062	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10063	break;
10064	}
10065
10066	case OP_IRET:
10067	{
10068	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10069	* instruction reference. */
10070	RTGCPTR GCPtrStack = 0;
10071	uint32_t uMask = 0xffff;
10072	uint16_t aIretFrame[3];
10073	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10074	{
10075	rc = VERR_EM_INTERPRETER;
10076	break;
10077	}
10078	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10079	&GCPtrStack);
10080	if (RT_SUCCESS(rc))
10081	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10082	if (RT_FAILURE(rc))
10083	{
10084	rc = VERR_EM_INTERPRETER;
10085	break;
10086	}
10087	pMixedCtx->eip = 0;
10088	pMixedCtx->ip = aIretFrame[0];
10089	pMixedCtx->cs.Sel = aIretFrame[1];
10090	pMixedCtx->cs.ValidSel = aIretFrame[1];
10091	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10092	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10093	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10094	pMixedCtx->sp += sizeof(aIretFrame);
10095	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
10096	\| HM_CHANGED_GUEST_RFLAGS;
10097	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10098	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10099	break;
10100	}
10101
10102	case OP_INT:
10103	{
10104	uint16_t uVector = pDis->Param1.uValue & 0xff;
10105	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10107	break;
10108	}
10109
10110	case OP_INTO:
10111	{
10112	if (pMixedCtx->eflags.Bits.u1OF)
10113	{
10114	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10116	}
10117	break;
10118	}
10119
10120	default:
10121	{
10122	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10123	EMCODETYPE_SUPERVISOR);
10124	rc = VBOXSTRICTRC_VAL(rc2);
10125	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
10126	Log4(("#GP rc=%Rrc\n", rc));
10127	break;
10128	}
10129	}
10130	}
10131	else
10132	rc = VERR_EM_INTERPRETER;
10133
10134	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10135	("#GP Unexpected rc=%Rrc\n", rc));
10136	return rc;
10137	}
10138
10139
10140	/**
10141	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10142	* the exception reported in the VMX transient structure back into the VM.
10143	*
10144	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
10145	* up-to-date.
10146	*/
10147	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10148	{
10149	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10150
10151	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10152	hmR0VmxCheckExitDueToEventDelivery(). */
10153	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10154	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10155	AssertRCReturn(rc, rc);
10156	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10157
10158	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10159	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10160	return VINF_SUCCESS;
10161	}
10162
10163
10164	/**
10165	* VM-exit exception handler for #PF (Page-fault exception).
10166	*/
10167	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10168	{
10169	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10170	PVM pVM = pVCpu->CTX_SUFF(pVM);
10171	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10172	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10173	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10174	AssertRCReturn(rc, rc);
10175
10176	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10177	if (pVM->hm.s.fNestedPaging)
10178	{
10179	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10180	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10181	{
10182	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10183	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10184	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10185	}
10186	else
10187	{
10188	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10189	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10190	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10191	}
10192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10193	return rc;
10194	}
10195	#else
10196	Assert(!pVM->hm.s.fNestedPaging);
10197	#endif
10198
10199	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10200	AssertRCReturn(rc, rc);
10201
10202	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10203	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10204
10205	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10206	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10207	(RTGCPTR)pVmxTransient->uExitQualification);
10208
10209	Log4(("#PF: rc=%Rrc\n", rc));
10210	if (rc == VINF_SUCCESS)
10211	{
10212	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10213	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10214	* memory? We don't update the whole state here... */
10215	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
10216	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
10217	TRPMResetTrap(pVCpu);
10218	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10219	return rc;
10220	}
10221	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10222	{
10223	if (!pVmxTransient->fVectoringPF)
10224	{
10225	/* It's a guest page fault and needs to be reflected to the guest. */
10226	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10227	TRPMResetTrap(pVCpu);
10228	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10229	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10230	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10231	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10232	}
10233	else
10234	{
10235	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10236	TRPMResetTrap(pVCpu);
10237	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10238	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10239	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10240	}
10241
10242	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10243	return VINF_SUCCESS;
10244	}
10245
10246	TRPMResetTrap(pVCpu);
10247	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10248	return rc;
10249	}
10250
10251	/** @} */
10252

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 47743

Download in other formats: