HMVMXR0.cpp@ 47719

Last change on this file since 47719 was 47719, checked in by vboxsync, 11 years ago
No MMIO2 aliasing when doing full IEM verification runs.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 426.1 KB

Line
1	/* $Id: HMVMXR0.cpp 47719 2013-08-14 10:34:44Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#endif
45
46
47	/*******************************************************************************
48	* Defined Constants And Macros *
49	*******************************************************************************/
50	#if defined(RT_ARCH_AMD64)
51	# define HMVMX_IS_64BIT_HOST_MODE() (true)
52	typedef RTHCUINTREG HMVMXHCUINTREG;
53	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54	extern "C" uint32_t g_fVMXIs64bitHost;
55	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56	typedef uint64_t HMVMXHCUINTREG;
57	#else
58	# define HMVMX_IS_64BIT_HOST_MODE() (false)
59	typedef RTHCUINTREG HMVMXHCUINTREG;
60	#endif
61
62	/** Use the function table. */
63	#define HMVMX_USE_FUNCTION_TABLE
64
65	/** Determine which tagged-TLB flush handler to use. */
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71	/** @name Updated-guest-state flags.
72	* @{ */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113	/** @} */
114
115	/** @name
116	* Flags to skip redundant reads of some common VMCS fields that are not part of
117	* the guest-CPU state but are in the transient structure.
118	*/
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125	/** @} */
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86).
129	*
130	* We need to intercept all exceptions manually (except #PF). #NM is also
131	* handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
132	* intercepted even in real-mode if we have Nested Paging support.
133	*/
134	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
135	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
136	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
137	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
138	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
139	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
140	\| RT_BIT(X86_XCPT_XF))
141
142	/**
143	* Exception bitmap mask for all contributory exceptions.
144	*
145	* Page fault is deliberately excluded here as it's conditional as to whether
146	* it's contributory or benign. Page faults are handled separately.
147	*/
148	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
149	\| RT_BIT(X86_XCPT_DE))
150
151	/** Maximum VM-instruction error number. */
152	#define HMVMX_INSTR_ERROR_MAX 28
153
154	/** Profiling macro. */
155	#ifdef HM_PROFILE_EXIT_DISPATCH
156	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
157	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
158	#else
159	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
160	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
161	#endif
162
163
164	/*******************************************************************************
165	* Structures and Typedefs *
166	*******************************************************************************/
167	/**
168	* VMX transient state.
169	*
170	* A state structure for holding miscellaneous information across
171	* VMX non-root operation and restored after the transition.
172	*/
173	typedef struct VMXTRANSIENT
174	{
175	/** The host's rflags/eflags. */
176	RTCCUINTREG uEFlags;
177	#if HC_ARCH_BITS == 32
178	uint32_t u32Alignment0;
179	#endif
180	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
181	uint64_t u64LStarMsr;
182	/** The guest's TPR value used for TPR shadowing. */
183	uint8_t u8GuestTpr;
184	/** Alignment. */
185	uint8_t abAlignment0[7];
186
187	/** The basic VM-exit reason. */
188	uint16_t uExitReason;
189	/** Alignment. */
190	uint16_t u16Alignment0;
191	/** The VM-exit interruption error code. */
192	uint32_t uExitIntrErrorCode;
193	/** The VM-exit exit qualification. */
194	uint64_t uExitQualification;
195
196	/** The VM-exit interruption-information field. */
197	uint32_t uExitIntrInfo;
198	/** The VM-exit instruction-length field. */
199	uint32_t cbInstr;
200	/** The VM-exit instruction-information field. */
201	union
202	{
203	/** Plain unsigned int representation. */
204	uint32_t u;
205	/** INS and OUTS information. */
206	struct
207	{
208	uint32_t u6Reserved0 : 6;
209	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
210	uint32_t u3AddrSize : 3;
211	uint32_t u5Reserved1 : 5;
212	/** The segment register (X86_SREG_XXX). */
213	uint32_t iSegReg : 3;
214	uint32_t uReserved2 : 14;
215	} StrIo;
216	} ExitInstrInfo;
217	/** Whether the VM-entry failed or not. */
218	bool fVMEntryFailed;
219	/** Alignment. */
220	uint8_t abAlignment1[3];
221
222	/** The VM-entry interruption-information field. */
223	uint32_t uEntryIntrInfo;
224	/** The VM-entry exception error code field. */
225	uint32_t uEntryXcptErrorCode;
226	/** The VM-entry instruction length field. */
227	uint32_t cbEntryInstr;
228
229	/** IDT-vectoring information field. */
230	uint32_t uIdtVectoringInfo;
231	/** IDT-vectoring error code. */
232	uint32_t uIdtVectoringErrorCode;
233
234	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
235	uint32_t fVmcsFieldsRead;
236	/** Whether TSC-offsetting should be setup before VM-entry. */
237	bool fUpdateTscOffsettingAndPreemptTimer;
238	/** Whether the VM-exit was caused by a page-fault during delivery of a
239	* contributory exception or a page-fault. */
240	bool fVectoringPF;
241	} VMXTRANSIENT;
242	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
243	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
244	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
245	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
246	/** Pointer to VMX transient state. */
247	typedef VMXTRANSIENT *PVMXTRANSIENT;
248
249
250	/**
251	* MSR-bitmap read permissions.
252	*/
253	typedef enum VMXMSREXITREAD
254	{
255	/** Reading this MSR causes a VM-exit. */
256	VMXMSREXIT_INTERCEPT_READ = 0xb,
257	/** Reading this MSR does not cause a VM-exit. */
258	VMXMSREXIT_PASSTHRU_READ
259	} VMXMSREXITREAD;
260
261	/**
262	* MSR-bitmap write permissions.
263	*/
264	typedef enum VMXMSREXITWRITE
265	{
266	/** Writing to this MSR causes a VM-exit. */
267	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
268	/** Writing to this MSR does not cause a VM-exit. */
269	VMXMSREXIT_PASSTHRU_WRITE
270	} VMXMSREXITWRITE;
271
272	/**
273	* VM-exit handler.
274	*
275	* @returns VBox status code.
276	* @param pVCpu Pointer to the VMCPU.
277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
278	* out-of-sync. Make sure to update the required
279	* fields before using them.
280	* @param pVmxTransient Pointer to the VMX-transient structure.
281	*/
282	#ifndef HMVMX_USE_FUNCTION_TABLE
283	typedef int FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	#else
285	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	/** Pointer to VM-exit handler. */
287	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
288	#endif
289
290
291	/*******************************************************************************
292	* Internal Functions *
293	*******************************************************************************/
294	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
295	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
296	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
297	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
298	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
299	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
300	#endif
301	#ifndef HMVMX_USE_FUNCTION_TABLE
302	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
303	# define HMVMX_EXIT_DECL static int
304	#else
305	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
306	#endif
307
308	/** @name VM-exit handlers.
309	* @{
310	*/
311	static FNVMEXITHANDLER hmR0VmxExitXcptOrNmi;
312	static FNVMEXITHANDLER hmR0VmxExitExtInt;
313	static FNVMEXITHANDLER hmR0VmxExitTripleFault;
314	static FNVMEXITHANDLER hmR0VmxExitInitSignal;
315	static FNVMEXITHANDLER hmR0VmxExitSipi;
316	static FNVMEXITHANDLER hmR0VmxExitIoSmi;
317	static FNVMEXITHANDLER hmR0VmxExitSmi;
318	static FNVMEXITHANDLER hmR0VmxExitIntWindow;
319	static FNVMEXITHANDLER hmR0VmxExitNmiWindow;
320	static FNVMEXITHANDLER hmR0VmxExitTaskSwitch;
321	static FNVMEXITHANDLER hmR0VmxExitCpuid;
322	static FNVMEXITHANDLER hmR0VmxExitGetsec;
323	static FNVMEXITHANDLER hmR0VmxExitHlt;
324	static FNVMEXITHANDLER hmR0VmxExitInvd;
325	static FNVMEXITHANDLER hmR0VmxExitInvlpg;
326	static FNVMEXITHANDLER hmR0VmxExitRdpmc;
327	static FNVMEXITHANDLER hmR0VmxExitRdtsc;
328	static FNVMEXITHANDLER hmR0VmxExitRsm;
329	static FNVMEXITHANDLER hmR0VmxExitSetPendingXcptUD;
330	static FNVMEXITHANDLER hmR0VmxExitMovCRx;
331	static FNVMEXITHANDLER hmR0VmxExitMovDRx;
332	static FNVMEXITHANDLER hmR0VmxExitIoInstr;
333	static FNVMEXITHANDLER hmR0VmxExitRdmsr;
334	static FNVMEXITHANDLER hmR0VmxExitWrmsr;
335	static FNVMEXITHANDLER hmR0VmxExitErrInvalidGuestState;
336	static FNVMEXITHANDLER hmR0VmxExitErrMsrLoad;
337	static FNVMEXITHANDLER hmR0VmxExitErrUndefined;
338	static FNVMEXITHANDLER hmR0VmxExitMwait;
339	static FNVMEXITHANDLER hmR0VmxExitMtf;
340	static FNVMEXITHANDLER hmR0VmxExitMonitor;
341	static FNVMEXITHANDLER hmR0VmxExitPause;
342	static FNVMEXITHANDLER hmR0VmxExitErrMachineCheck;
343	static FNVMEXITHANDLER hmR0VmxExitTprBelowThreshold;
344	static FNVMEXITHANDLER hmR0VmxExitApicAccess;
345	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
346	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
347	static FNVMEXITHANDLER hmR0VmxExitEptViolation;
348	static FNVMEXITHANDLER hmR0VmxExitEptMisconfig;
349	static FNVMEXITHANDLER hmR0VmxExitRdtscp;
350	static FNVMEXITHANDLER hmR0VmxExitPreemptTimer;
351	static FNVMEXITHANDLER hmR0VmxExitWbinvd;
352	static FNVMEXITHANDLER hmR0VmxExitXsetbv;
353	static FNVMEXITHANDLER hmR0VmxExitRdrand;
354	static FNVMEXITHANDLER hmR0VmxExitInvpcid;
355	/** @} */
356
357	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
358	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
359	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
360	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
361	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
362	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
363	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
365
366	/*******************************************************************************
367	* Global Variables *
368	*******************************************************************************/
369	#ifdef HMVMX_USE_FUNCTION_TABLE
370
371	/**
372	* VMX_EXIT dispatch table.
373	*/
374	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
375	{
376	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
377	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
378	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
379	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
380	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
381	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
382	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
383	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
384	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
385	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
386	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
387	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
388	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
389	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
390	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
391	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
392	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
393	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
394	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
395	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
396	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
397	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
398	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
399	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
400	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
401	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
402	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
403	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
404	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
405	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
406	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
407	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
408	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
409	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
410	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
411	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
412	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
413	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
414	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
415	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
416	/* 40 UNDEFINED */ hmR0VmxExitPause,
417	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
418	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
419	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
420	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
421	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
422	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
423	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
424	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
425	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
426	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
427	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
428	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
429	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
430	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
431	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
432	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
433	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
434	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
435	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
436	};
437	#endif /* HMVMX_USE_FUNCTION_TABLE */
438
439	#ifdef VBOX_STRICT
440	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
441	{
442	/* 0 */ "(Not Used)",
443	/* 1 */ "VMCALL executed in VMX root operation.",
444	/* 2 */ "VMCLEAR with invalid physical address.",
445	/* 3 */ "VMCLEAR with VMXON pointer.",
446	/* 4 */ "VMLAUNCH with non-clear VMCS.",
447	/* 5 */ "VMRESUME with non-launched VMCS.",
448	/* 6 */ "VMRESUME after VMXOFF",
449	/* 7 */ "VM entry with invalid control fields.",
450	/* 8 */ "VM entry with invalid host state fields.",
451	/* 9 */ "VMPTRLD with invalid physical address.",
452	/* 10 */ "VMPTRLD with VMXON pointer.",
453	/* 11 */ "VMPTRLD with incorrect revision identifier.",
454	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
455	/* 13 */ "VMWRITE to read-only VMCS component.",
456	/* 14 */ "(Not Used)",
457	/* 15 */ "VMXON executed in VMX root operation.",
458	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
459	/* 17 */ "VM entry with non-launched executing VMCS.",
460	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
461	/* 19 */ "VMCALL with non-clear VMCS.",
462	/* 20 */ "VMCALL with invalid VM-exit control fields.",
463	/* 21 */ "(Not Used)",
464	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
465	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
466	/* 24 */ "VMCALL with invalid SMM-monitor features.",
467	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
468	/* 26 */ "VM entry with events blocked by MOV SS.",
469	/* 27 */ "(Not Used)",
470	/* 28 */ "Invalid operand to INVEPT/INVVPID."
471	};
472	#endif /* VBOX_STRICT */
473
474
475
476	/**
477	* Updates the VM's last error record. If there was a VMX instruction error,
478	* reads the error data from the VMCS and updates VCPU's last error record as
479	* well.
480	*
481	* @param pVM Pointer to the VM.
482	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
483	* VERR_VMX_UNABLE_TO_START_VM or
484	* VERR_VMX_INVALID_VMCS_FIELD).
485	* @param rc The error code.
486	*/
487	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
488	{
489	AssertPtr(pVM);
490	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
491	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
492	{
493	AssertPtrReturnVoid(pVCpu);
494	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
495	}
496	pVM->hm.s.lLastError = rc;
497	}
498
499
500	/**
501	* Reads the VM-entry interruption-information field from the VMCS into the VMX
502	* transient structure.
503	*
504	* @returns VBox status code.
505	* @param pVmxTransient Pointer to the VMX transient structure.
506	*
507	* @remarks No-long-jump zone!!!
508	*/
509	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
510	{
511	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
512	AssertRCReturn(rc, rc);
513	return VINF_SUCCESS;
514	}
515
516
517	/**
518	* Reads the VM-entry exception error code field from the VMCS into
519	* the VMX transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVmxTransient Pointer to the VMX transient structure.
523	*
524	* @remarks No-long-jump zone!!!
525	*/
526	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
527	{
528	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
529	AssertRCReturn(rc, rc);
530	return VINF_SUCCESS;
531	}
532
533
534	/**
535	* Reads the VM-entry exception error code field from the VMCS into
536	* the VMX transient structure.
537	*
538	* @returns VBox status code.
539	* @param pVCpu Pointer to the VMCPU.
540	* @param pVmxTransient Pointer to the VMX transient structure.
541	*
542	* @remarks No-long-jump zone!!!
543	*/
544	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
545	{
546	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
547	AssertRCReturn(rc, rc);
548	return VINF_SUCCESS;
549	}
550
551
552	/**
553	* Reads the VM-exit interruption-information field from the VMCS into the VMX
554	* transient structure.
555	*
556	* @returns VBox status code.
557	* @param pVCpu Pointer to the VMCPU.
558	* @param pVmxTransient Pointer to the VMX transient structure.
559	*/
560	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
561	{
562	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
563	{
564	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
565	AssertRCReturn(rc, rc);
566	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
567	}
568	return VINF_SUCCESS;
569	}
570
571
572	/**
573	* Reads the VM-exit interruption error code from the VMCS into the VMX
574	* transient structure.
575	*
576	* @returns VBox status code.
577	* @param pVCpu Pointer to the VMCPU.
578	* @param pVmxTransient Pointer to the VMX transient structure.
579	*/
580	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
581	{
582	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
583	{
584	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
585	AssertRCReturn(rc, rc);
586	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
587	}
588	return VINF_SUCCESS;
589	}
590
591
592	/**
593	* Reads the VM-exit instruction length field from the VMCS into the VMX
594	* transient structure.
595	*
596	* @returns VBox status code.
597	* @param pVCpu Pointer to the VMCPU.
598	* @param pVmxTransient Pointer to the VMX transient structure.
599	*/
600	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
601	{
602	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
603	{
604	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
605	AssertRCReturn(rc, rc);
606	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
607	}
608	return VINF_SUCCESS;
609	}
610
611
612	/**
613	* Reads the VM-exit instruction-information field from the VMCS into
614	* the VMX transient structure.
615	*
616	* @returns VBox status code.
617	* @param pVCpu The cross context per CPU structure.
618	* @param pVmxTransient Pointer to the VMX transient structure.
619	*/
620	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
621	{
622	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
623	{
624	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
625	AssertRCReturn(rc, rc);
626	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
627	}
628	return VINF_SUCCESS;
629	}
630
631
632	/**
633	* Reads the exit qualification from the VMCS into the VMX transient structure.
634	*
635	* @returns VBox status code.
636	* @param pVCpu Pointer to the VMCPU.
637	* @param pVmxTransient Pointer to the VMX transient structure.
638	*/
639	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
640	{
641	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
642	{
643	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
644	AssertRCReturn(rc, rc);
645	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
646	}
647	return VINF_SUCCESS;
648	}
649
650
651	/**
652	* Reads the IDT-vectoring information field from the VMCS into the VMX
653	* transient structure.
654	*
655	* @returns VBox status code.
656	* @param pVmxTransient Pointer to the VMX transient structure.
657	*
658	* @remarks No-long-jump zone!!!
659	*/
660	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
661	{
662	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
663	{
664	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
665	AssertRCReturn(rc, rc);
666	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
667	}
668	return VINF_SUCCESS;
669	}
670
671
672	/**
673	* Reads the IDT-vectoring error code from the VMCS into the VMX
674	* transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVmxTransient Pointer to the VMX transient structure.
678	*/
679	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
680	{
681	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
682	{
683	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
684	AssertRCReturn(rc, rc);
685	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
686	}
687	return VINF_SUCCESS;
688	}
689
690
691	/**
692	* Enters VMX root mode operation on the current CPU.
693	*
694	* @returns VBox status code.
695	* @param pVM Pointer to the VM (optional, can be NULL, after
696	* a resume).
697	* @param HCPhysCpuPage Physical address of the VMXON region.
698	* @param pvCpuPage Pointer to the VMXON region.
699	*/
700	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
701	{
702	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
703	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
704	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
705
706	if (pVM)
707	{
708	/* Write the VMCS revision dword to the VMXON region. */
709	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
710	}
711
712	/* Enable the VMX bit in CR4 if necessary. */
713	RTCCUINTREG uCr4 = ASMGetCR4();
714	if (!(uCr4 & X86_CR4_VMXE))
715	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
716
717	/* Enter VMX root mode. */
718	int rc = VMXEnable(HCPhysCpuPage);
719	if (RT_FAILURE(rc))
720	ASMSetCR4(uCr4);
721
722	return rc;
723	}
724
725
726	/**
727	* Exits VMX root mode operation on the current CPU.
728	*
729	* @returns VBox status code.
730	*/
731	static int hmR0VmxLeaveRootMode(void)
732	{
733	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
734
735	/* If we're for some reason not in VMX root mode, then don't leave it. */
736	RTCCUINTREG uHostCR4 = ASMGetCR4();
737	if (uHostCR4 & X86_CR4_VMXE)
738	{
739	/* Exit VMX root mode and clear the VMX bit in CR4. */
740	VMXDisable();
741	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
742	return VINF_SUCCESS;
743	}
744
745	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
746	}
747
748
749	/**
750	* Allocates and maps one physically contiguous page. The allocated page is
751	* zero'd out. (Used by various VT-x structures).
752	*
753	* @returns IPRT status code.
754	* @param pMemObj Pointer to the ring-0 memory object.
755	* @param ppVirt Where to store the virtual address of the
756	* allocation.
757	* @param pPhys Where to store the physical address of the
758	* allocation.
759	*/
760	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
761	{
762	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
763	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
764	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
765
766	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
767	if (RT_FAILURE(rc))
768	return rc;
769	ppVirt = RTR0MemObjAddress(pMemObj);
770	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
771	ASMMemZero32(*ppVirt, PAGE_SIZE);
772	return VINF_SUCCESS;
773	}
774
775
776	/**
777	* Frees and unmaps an allocated physical page.
778	*
779	* @param pMemObj Pointer to the ring-0 memory object.
780	* @param ppVirt Where to re-initialize the virtual address of
781	* allocation as 0.
782	* @param pHCPhys Where to re-initialize the physical address of the
783	* allocation as 0.
784	*/
785	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
786	{
787	AssertPtr(pMemObj);
788	AssertPtr(ppVirt);
789	AssertPtr(pHCPhys);
790	if (*pMemObj != NIL_RTR0MEMOBJ)
791	{
792	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
793	AssertRC(rc);
794	*pMemObj = NIL_RTR0MEMOBJ;
795	*ppVirt = 0;
796	*pHCPhys = 0;
797	}
798	}
799
800
801	/**
802	* Worker function to free VT-x related structures.
803	*
804	* @returns IPRT status code.
805	* @param pVM Pointer to the VM.
806	*/
807	static void hmR0VmxStructsFree(PVM pVM)
808	{
809	for (VMCPUID i = 0; i < pVM->cCpus; i++)
810	{
811	PVMCPU pVCpu = &pVM->aCpus[i];
812	AssertPtr(pVCpu);
813
814	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
815	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
816	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
817	#endif
818
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
820	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
821
822	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
823	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
824	}
825
826	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
827	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
828	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
829	#endif
830	}
831
832
833	/**
834	* Worker function to allocate VT-x related VM structures.
835	*
836	* @returns IPRT status code.
837	* @param pVM Pointer to the VM.
838	*/
839	static int hmR0VmxStructsAlloc(PVM pVM)
840	{
841	/*
842	* Initialize members up-front so we can cleanup properly on allocation failure.
843	*/
844	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
845	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
846	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
847	pVM->hm.s.vmx.HCPhys##a_Name = 0;
848
849	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
850	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
851	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
852	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
853
854	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
855	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
856	#endif
857	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
858
859	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
860	for (VMCPUID i = 0; i < pVM->cCpus; i++)
861	{
862	PVMCPU pVCpu = &pVM->aCpus[i];
863	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
864	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
865	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
866	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
867	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
868	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
869	#endif
870	}
871	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
872	#undef VMXLOCAL_INIT_VM_MEMOBJ
873
874	/*
875	* Allocate all the VT-x structures.
876	*/
877	int rc = VINF_SUCCESS;
878	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
879	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
883	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
884	#endif
885
886	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
887	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
888	{
889	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
890	&pVM->hm.s.vmx.HCPhysApicAccess);
891	if (RT_FAILURE(rc))
892	goto cleanup;
893	}
894
895	/*
896	* Initialize per-VCPU VT-x structures.
897	*/
898	for (VMCPUID i = 0; i < pVM->cCpus; i++)
899	{
900	PVMCPU pVCpu = &pVM->aCpus[i];
901	AssertPtr(pVCpu);
902
903	/* Allocate the VM control structure (VMCS). */
904	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
905	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
906	if (RT_FAILURE(rc))
907	goto cleanup;
908
909	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
910	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
911	{
912	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
913	&pVCpu->hm.s.vmx.HCPhysVirtApic);
914	if (RT_FAILURE(rc))
915	goto cleanup;
916	}
917
918	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
919	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
920	{
921	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
922	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
923	if (RT_FAILURE(rc))
924	goto cleanup;
925	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
926	}
927
928	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
929	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
930	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
931	if (RT_FAILURE(rc))
932	goto cleanup;
933
934	/* Allocate the VM-exit MSR-load page for the host MSRs. */
935	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
936	if (RT_FAILURE(rc))
937	goto cleanup;
938	#endif
939	}
940
941	return VINF_SUCCESS;
942
943	cleanup:
944	hmR0VmxStructsFree(pVM);
945	return rc;
946	}
947
948
949	/**
950	* Does global VT-x initialization (called during module initialization).
951	*
952	* @returns VBox status code.
953	*/
954	VMMR0DECL(int) VMXR0GlobalInit(void)
955	{
956	#ifdef HMVMX_USE_FUNCTION_TABLE
957	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
958	# ifdef VBOX_STRICT
959	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
960	Assert(g_apfnVMExitHandlers[i]);
961	# endif
962	#endif
963	return VINF_SUCCESS;
964	}
965
966
967	/**
968	* Does global VT-x termination (called during module termination).
969	*/
970	VMMR0DECL(void) VMXR0GlobalTerm()
971	{
972	/* Nothing to do currently. */
973	}
974
975
976	/**
977	* Sets up and activates VT-x on the current CPU.
978	*
979	* @returns VBox status code.
980	* @param pCpu Pointer to the global CPU info struct.
981	* @param pVM Pointer to the VM (can be NULL after a host resume
982	* operation).
983	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
984	* fEnabledByHost is true).
985	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
986	* @a fEnabledByHost is true).
987	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
988	* enable VT-x on the host.
989	*/
990	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
991	{
992	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
993	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
994
995	if (!fEnabledByHost)
996	{
997	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
998	if (RT_FAILURE(rc))
999	return rc;
1000	}
1001
1002	/*
1003	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
1004	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
1005	* each time while reusing a VPID after hitting the MaxASID limit once.
1006	*/
1007	if ( pVM
1008	&& pVM->hm.s.fNestedPaging)
1009	{
1010	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
1011	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
1012	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1013	pCpu->fFlushAsidBeforeUse = false;
1014	}
1015	else
1016	{
1017	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
1018	* without Nested Paging triggered this function) we still have the risk
1019	* of potentially running with stale TLB-entries from other hypervisors
1020	* when later we use a VM with NestedPaging. To fix this properly we will
1021	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
1022	* 'vmx_ept_vpid_caps' from it. Sigh. */
1023	pCpu->fFlushAsidBeforeUse = true;
1024	}
1025
1026	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1027	++pCpu->cTlbFlushes;
1028
1029	return VINF_SUCCESS;
1030	}
1031
1032
1033	/**
1034	* Deactivates VT-x on the current CPU.
1035	*
1036	* @returns VBox status code.
1037	* @param pCpu Pointer to the global CPU info struct.
1038	* @param pvCpuPage Pointer to the VMXON region.
1039	* @param HCPhysCpuPage Physical address of the VMXON region.
1040	*
1041	* @remarks This function should never be called when SUPR0EnableVTx() or
1042	* similar was used to enable VT-x on the host.
1043	*/
1044	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1045	{
1046	NOREF(pCpu);
1047	NOREF(pvCpuPage);
1048	NOREF(HCPhysCpuPage);
1049
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051	return hmR0VmxLeaveRootMode();
1052	}
1053
1054
1055	/**
1056	* Sets the permission bits for the specified MSR in the MSR bitmap.
1057	*
1058	* @param pVCpu Pointer to the VMCPU.
1059	* @param uMSR The MSR value.
1060	* @param enmRead Whether reading this MSR causes a VM-exit.
1061	* @param enmWrite Whether writing this MSR causes a VM-exit.
1062	*/
1063	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1064	{
1065	int32_t iBit;
1066	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1067
1068	/*
1069	* Layout:
1070	* 0x000 - 0x3ff - Low MSR read bits
1071	* 0x400 - 0x7ff - High MSR read bits
1072	* 0x800 - 0xbff - Low MSR write bits
1073	* 0xc00 - 0xfff - High MSR write bits
1074	*/
1075	if (uMsr <= 0x00001FFF)
1076	iBit = uMsr;
1077	else if ( uMsr >= 0xC0000000
1078	&& uMsr <= 0xC0001FFF)
1079	{
1080	iBit = (uMsr - 0xC0000000);
1081	pbMsrBitmap += 0x400;
1082	}
1083	else
1084	{
1085	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1086	return;
1087	}
1088
1089	Assert(iBit <= 0x1fff);
1090	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1091	ASMBitSet(pbMsrBitmap, iBit);
1092	else
1093	ASMBitClear(pbMsrBitmap, iBit);
1094
1095	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1096	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1097	else
1098	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1099	}
1100
1101
1102	/**
1103	* Flushes the TLB using EPT.
1104	*
1105	* @returns VBox status code.
1106	* @param pVM Pointer to the VM.
1107	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1108	* enmFlush).
1109	* @param enmFlush Type of flush.
1110	*/
1111	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1112	{
1113	AssertPtr(pVM);
1114	Assert(pVM->hm.s.fNestedPaging);
1115
1116	uint64_t descriptor[2];
1117	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1118	descriptor[0] = 0;
1119	else
1120	{
1121	Assert(pVCpu);
1122	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1123	}
1124	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1125
1126	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1127	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1128	rc));
1129	if ( RT_SUCCESS(rc)
1130	&& pVCpu)
1131	{
1132	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1133	}
1134	}
1135
1136
1137	/**
1138	* Flushes the TLB using VPID.
1139	*
1140	* @returns VBox status code.
1141	* @param pVM Pointer to the VM.
1142	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1143	* enmFlush).
1144	* @param enmFlush Type of flush.
1145	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1146	* on @a enmFlush).
1147	*/
1148	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1149	{
1150	AssertPtr(pVM);
1151	Assert(pVM->hm.s.vmx.fVpid);
1152
1153	uint64_t descriptor[2];
1154	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1155	{
1156	descriptor[0] = 0;
1157	descriptor[1] = 0;
1158	}
1159	else
1160	{
1161	AssertPtr(pVCpu);
1162	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1163	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1164	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1165	descriptor[1] = GCPtr;
1166	}
1167
1168	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1169	AssertMsg(rc == VINF_SUCCESS,
1170	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1171	if ( RT_SUCCESS(rc)
1172	&& pVCpu)
1173	{
1174	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1175	}
1176	}
1177
1178
1179	/**
1180	* Invalidates a guest page by guest virtual address. Only relevant for
1181	* EPT/VPID, otherwise there is nothing really to invalidate.
1182	*
1183	* @returns VBox status code.
1184	* @param pVM Pointer to the VM.
1185	* @param pVCpu Pointer to the VMCPU.
1186	* @param GCVirt Guest virtual address of the page to invalidate.
1187	*/
1188	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1189	{
1190	AssertPtr(pVM);
1191	AssertPtr(pVCpu);
1192	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1193
1194	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1195	if (!fFlushPending)
1196	{
1197	/*
1198	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1199	* See @bugref{6043} and @bugref{6177}.
1200	*
1201	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1202	* function maybe called in a loop with individual addresses.
1203	*/
1204	if (pVM->hm.s.vmx.fVpid)
1205	{
1206	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1207	{
1208	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1209	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1210	}
1211	else
1212	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1213	}
1214	else if (pVM->hm.s.fNestedPaging)
1215	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1216	}
1217
1218	return VINF_SUCCESS;
1219	}
1220
1221
1222	/**
1223	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1224	* otherwise there is nothing really to invalidate.
1225	*
1226	* @returns VBox status code.
1227	* @param pVM Pointer to the VM.
1228	* @param pVCpu Pointer to the VMCPU.
1229	* @param GCPhys Guest physical address of the page to invalidate.
1230	*/
1231	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1232	{
1233	LogFlowFunc(("%RGp\n", GCPhys));
1234
1235	/*
1236	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1237	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1238	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1239	*/
1240	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1241	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1242	return VINF_SUCCESS;
1243	}
1244
1245
1246	/**
1247	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1248	* case where neither EPT nor VPID is supported by the CPU.
1249	*
1250	* @param pVM Pointer to the VM.
1251	* @param pVCpu Pointer to the VMCPU.
1252	*
1253	* @remarks Called with interrupts disabled.
1254	*/
1255	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1256	{
1257	NOREF(pVM);
1258	AssertPtr(pVCpu);
1259	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1260	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1261
1262	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1263	AssertPtr(pCpu);
1264
1265	pVCpu->hm.s.TlbShootdown.cPages = 0;
1266	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1267	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1268	pVCpu->hm.s.fForceTLBFlush = false;
1269	return;
1270	}
1271
1272
1273	/**
1274	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1275	*
1276	* @param pVM Pointer to the VM.
1277	* @param pVCpu Pointer to the VMCPU.
1278	* @remarks All references to "ASID" in this function pertains to "VPID" in
1279	* Intel's nomenclature. The reason is, to avoid confusion in compare
1280	* statements since the host-CPU copies are named "ASID".
1281	*
1282	* @remarks Called with interrupts disabled.
1283	*/
1284	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1285	{
1286	#ifdef VBOX_WITH_STATISTICS
1287	bool fTlbFlushed = false;
1288	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1289	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1290	if (!fTlbFlushed) \
1291	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1292	} while (0)
1293	#else
1294	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1295	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1296	#endif
1297
1298	AssertPtr(pVM);
1299	AssertPtr(pVCpu);
1300	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1301	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1302	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1303
1304	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1305	AssertPtr(pCpu);
1306
1307	/*
1308	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1309	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1310	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1311	*/
1312	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1313	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1314	{
1315	++pCpu->uCurrentAsid;
1316	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1317	{
1318	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1319	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1320	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1321	}
1322
1323	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1324	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1325	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1326
1327	/*
1328	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1329	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1330	*/
1331	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1332	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1333	HMVMX_SET_TAGGED_TLB_FLUSHED();
1334	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1335	}
1336
1337	/* Check for explicit TLB shootdowns. */
1338	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1339	{
1340	/*
1341	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1342	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1343	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1344	* but not guest-physical mappings.
1345	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1346	*/
1347	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1348	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1349	HMVMX_SET_TAGGED_TLB_FLUSHED();
1350	}
1351
1352	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1353	* not be executed. See hmQueueInvlPage() where it is commented
1354	* out. Support individual entry flushing someday. */
1355	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1356	{
1357	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1358
1359	/*
1360	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1361	* as supported by the CPU.
1362	*/
1363	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1364	{
1365	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1366	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1367	}
1368	else
1369	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1370
1371	HMVMX_SET_TAGGED_TLB_FLUSHED();
1372	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1373	}
1374
1375	pVCpu->hm.s.TlbShootdown.cPages = 0;
1376	pVCpu->hm.s.fForceTLBFlush = false;
1377
1378	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1379
1380	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1381	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1382	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1383	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1384	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1385	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1386	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1387	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1388
1389	/* Update VMCS with the VPID. */
1390	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1391	AssertRC(rc);
1392
1393	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1394	}
1395
1396
1397	/**
1398	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1399	*
1400	* @returns VBox status code.
1401	* @param pVM Pointer to the VM.
1402	* @param pVCpu Pointer to the VMCPU.
1403	*
1404	* @remarks Called with interrupts disabled.
1405	*/
1406	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1407	{
1408	AssertPtr(pVM);
1409	AssertPtr(pVCpu);
1410	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1411	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1412
1413	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1414	AssertPtr(pCpu);
1415
1416	/*
1417	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1418	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1419	*/
1420	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1421	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1422	{
1423	pVCpu->hm.s.fForceTLBFlush = true;
1424	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1425	}
1426
1427	/* Check for explicit TLB shootdown flushes. */
1428	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1429	{
1430	pVCpu->hm.s.fForceTLBFlush = true;
1431	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1432	}
1433
1434	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1435	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1436
1437	if (pVCpu->hm.s.fForceTLBFlush)
1438	{
1439	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1440	pVCpu->hm.s.fForceTLBFlush = false;
1441	}
1442	else
1443	{
1444	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1445	* not be executed. See hmQueueInvlPage() where it is commented
1446	* out. Support individual entry flushing someday. */
1447	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1448	{
1449	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1450	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1451	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1452	}
1453	else
1454	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1455	}
1456
1457	pVCpu->hm.s.TlbShootdown.cPages = 0;
1458	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1459	}
1460
1461
1462	/**
1463	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1464	*
1465	* @returns VBox status code.
1466	* @param pVM Pointer to the VM.
1467	* @param pVCpu Pointer to the VMCPU.
1468	*
1469	* @remarks Called with interrupts disabled.
1470	*/
1471	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1472	{
1473	AssertPtr(pVM);
1474	AssertPtr(pVCpu);
1475	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1476	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1477
1478	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1479
1480	/*
1481	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1482	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1483	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1484	*/
1485	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1486	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1487	{
1488	pVCpu->hm.s.fForceTLBFlush = true;
1489	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1490	}
1491
1492	/* Check for explicit TLB shootdown flushes. */
1493	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1494	{
1495	/*
1496	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1497	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1498	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1499	*/
1500	pVCpu->hm.s.fForceTLBFlush = true;
1501	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1502	}
1503
1504	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1505	if (pVCpu->hm.s.fForceTLBFlush)
1506	{
1507	++pCpu->uCurrentAsid;
1508	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1509	{
1510	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1511	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1512	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1513	}
1514
1515	pVCpu->hm.s.fForceTLBFlush = false;
1516	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1517	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1518	if (pCpu->fFlushAsidBeforeUse)
1519	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1520	}
1521	else
1522	{
1523	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1524	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1525	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1526	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1527
1528	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1529	* not be executed. See hmQueueInvlPage() where it is commented
1530	* out. Support individual entry flushing someday. */
1531	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1532	{
1533	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1534	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1535	{
1536	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1537	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1538	}
1539	else
1540	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1541	}
1542	else
1543	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1544	}
1545
1546	pVCpu->hm.s.TlbShootdown.cPages = 0;
1547	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1548
1549	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1550	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1551	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1552	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1553	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1554	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1555
1556	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1557	AssertRC(rc);
1558	}
1559
1560
1561	/**
1562	* Flushes the guest TLB entry based on CPU capabilities.
1563	*
1564	* @param pVCpu Pointer to the VMCPU.
1565	*/
1566	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1567	{
1568	PVM pVM = pVCpu->CTX_SUFF(pVM);
1569	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1570	{
1571	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1572	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1573	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1574	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1575	default:
1576	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1577	break;
1578	}
1579	}
1580
1581
1582	/**
1583	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1584	* TLB entries from the host TLB before VM-entry.
1585	*
1586	* @returns VBox status code.
1587	* @param pVM Pointer to the VM.
1588	*/
1589	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1590	{
1591	/*
1592	* Determine optimal flush type for Nested Paging.
1593	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1594	* guest execution (see hmR3InitFinalizeR0()).
1595	*/
1596	if (pVM->hm.s.fNestedPaging)
1597	{
1598	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1599	{
1600	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1601	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1602	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1603	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1604	else
1605	{
1606	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1607	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1608	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1609	}
1610
1611	/* Make sure the write-back cacheable memory type for EPT is supported. */
1612	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1613	{
1614	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1615	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1616	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1617	}
1618	}
1619	else
1620	{
1621	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1622	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1623	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1624	}
1625	}
1626
1627	/*
1628	* Determine optimal flush type for VPID.
1629	*/
1630	if (pVM->hm.s.vmx.fVpid)
1631	{
1632	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1633	{
1634	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1635	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1636	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1637	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1638	else
1639	{
1640	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1641	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1642	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1643	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1644	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1645	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1646	pVM->hm.s.vmx.fVpid = false;
1647	}
1648	}
1649	else
1650	{
1651	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1652	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1653	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1654	pVM->hm.s.vmx.fVpid = false;
1655	}
1656	}
1657
1658	/*
1659	* Setup the handler for flushing tagged-TLBs.
1660	*/
1661	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1662	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1663	else if (pVM->hm.s.fNestedPaging)
1664	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1665	else if (pVM->hm.s.vmx.fVpid)
1666	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1667	else
1668	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1669	return VINF_SUCCESS;
1670	}
1671
1672
1673	/**
1674	* Sets up pin-based VM-execution controls in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVM Pointer to the VM.
1678	* @param pVCpu Pointer to the VMCPU.
1679	*/
1680	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1681	{
1682	AssertPtr(pVM);
1683	AssertPtr(pVCpu);
1684
1685	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1686	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1687
1688	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1689	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1690	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1691
1692	/* Enable the VMX preemption timer. */
1693	if (pVM->hm.s.vmx.fUsePreemptTimer)
1694	{
1695	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1696	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1697	}
1698
1699	if ((val & zap) != val)
1700	{
1701	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1702	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1703	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1704	}
1705
1706	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1707	AssertRCReturn(rc, rc);
1708
1709	/* Update VCPU with the currently set pin-based VM-execution controls. */
1710	pVCpu->hm.s.vmx.u32PinCtls = val;
1711	return rc;
1712	}
1713
1714
1715	/**
1716	* Sets up processor-based VM-execution controls in the VMCS.
1717	*
1718	* @returns VBox status code.
1719	* @param pVM Pointer to the VM.
1720	* @param pVMCPU Pointer to the VMCPU.
1721	*/
1722	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1723	{
1724	AssertPtr(pVM);
1725	AssertPtr(pVCpu);
1726
1727	int rc = VERR_INTERNAL_ERROR_5;
1728	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1729	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1730
1731	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1732	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1733	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1734	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1735	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1736	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1737	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1738
1739	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1740	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1741	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1742	{
1743	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1744	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1745	}
1746
1747	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1748	if (!pVM->hm.s.fNestedPaging)
1749	{
1750	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1751	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1752	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1753	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1754	}
1755
1756	/* Use TPR shadowing if supported by the CPU. */
1757	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1758	{
1759	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1760	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1761	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1762	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1763	AssertRCReturn(rc, rc);
1764
1765	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1766	/* CR8 writes causes a VM-exit based on TPR threshold. */
1767	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1768	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1769	}
1770	else
1771	{
1772	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1773	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1774	}
1775
1776	/* Use MSR-bitmaps if supported by the CPU. */
1777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1778	{
1779	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1780
1781	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1782	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1783	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1784	AssertRCReturn(rc, rc);
1785
1786	/*
1787	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1788	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1789	*/
1790	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1791	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1792	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1793	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1794	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1795	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1796	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1797	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1798	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1799	}
1800
1801	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1802	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1803	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1804
1805	if ((val & zap) != val)
1806	{
1807	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1808	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1809	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1810	}
1811
1812	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1813	AssertRCReturn(rc, rc);
1814
1815	/* Update VCPU with the currently set processor-based VM-execution controls. */
1816	pVCpu->hm.s.vmx.u32ProcCtls = val;
1817
1818	/*
1819	* Secondary processor-based VM-execution controls.
1820	*/
1821	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1822	{
1823	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1824	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1825
1826	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1827	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1828
1829	if (pVM->hm.s.fNestedPaging)
1830	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1831	else
1832	{
1833	/*
1834	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1835	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1836	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1837	*/
1838	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1839	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1840	}
1841
1842	if (pVM->hm.s.vmx.fVpid)
1843	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1844
1845	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1846	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1847
1848	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1849	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1850	* done dynamically. */
1851	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1852	{
1853	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1854	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1855	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1856	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1857	AssertRCReturn(rc, rc);
1858	}
1859
1860	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1861	{
1862	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1864	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1865	}
1866
1867	if ((val & zap) != val)
1868	{
1869	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1870	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1871	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1872	}
1873
1874	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1875	AssertRCReturn(rc, rc);
1876
1877	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1878	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1879	}
1880	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1881	{
1882	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1883	"available\n"));
1884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1885	}
1886
1887	return VINF_SUCCESS;
1888	}
1889
1890
1891	/**
1892	* Sets up miscellaneous (everything other than Pin & Processor-based
1893	* VM-execution) control fields in the VMCS.
1894	*
1895	* @returns VBox status code.
1896	* @param pVM Pointer to the VM.
1897	* @param pVCpu Pointer to the VMCPU.
1898	*/
1899	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1900	{
1901	AssertPtr(pVM);
1902	AssertPtr(pVCpu);
1903
1904	int rc = VERR_GENERAL_FAILURE;
1905
1906	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1907	#if 0
1908	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1909	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1910	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1911
1912	/*
1913	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1914	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1915	* We thus use the exception bitmap to control it rather than use both.
1916	*/
1917	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1918	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1919
1920	/** @todo Explore possibility of using IO-bitmaps. */
1921	/* All IO & IOIO instructions cause VM-exits. */
1922	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1923	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1924
1925	/* Initialize the MSR-bitmap area. */
1926	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1927	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1928	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1929	#endif
1930
1931	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1932	/* Setup MSR autoloading/storing. */
1933	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1934	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1935	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1936	AssertRCReturn(rc, rc);
1937	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1938	AssertRCReturn(rc, rc);
1939
1940	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1941	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1942	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1943	AssertRCReturn(rc, rc);
1944	#endif
1945
1946	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1947	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1948	AssertRCReturn(rc, rc);
1949
1950	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1951	#if 0
1952	/* Setup debug controls */
1953	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1954	AssertRCReturn(rc, rc);
1955	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1956	AssertRCReturn(rc, rc);
1957	#endif
1958
1959	return rc;
1960	}
1961
1962
1963	/**
1964	* Sets up the initial exception bitmap in the VMCS based on static conditions
1965	* (i.e. conditions that cannot ever change at runtime).
1966	*
1967	* @returns VBox status code.
1968	* @param pVM Pointer to the VM.
1969	* @param pVCpu Pointer to the VMCPU.
1970	*/
1971	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1972	{
1973	AssertPtr(pVM);
1974	AssertPtr(pVCpu);
1975
1976	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1977
1978	uint32_t u32XcptBitmap = 0;
1979
1980	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1981	if (!pVM->hm.s.fNestedPaging)
1982	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1983
1984	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1985	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1986	AssertRCReturn(rc, rc);
1987	return rc;
1988	}
1989
1990
1991	/**
1992	* Sets up the initial guest-state mask. The guest-state mask is consulted
1993	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1994	* for the nested virtualization case (as it would cause a VM-exit).
1995	*
1996	* @param pVCpu Pointer to the VMCPU.
1997	*/
1998	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1999	{
2000	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2001	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2002	return VINF_SUCCESS;
2003	}
2004
2005
2006	/**
2007	* Does per-VM VT-x initialization.
2008	*
2009	* @returns VBox status code.
2010	* @param pVM Pointer to the VM.
2011	*/
2012	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2013	{
2014	LogFlowFunc(("pVM=%p\n", pVM));
2015
2016	int rc = hmR0VmxStructsAlloc(pVM);
2017	if (RT_FAILURE(rc))
2018	{
2019	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2020	return rc;
2021	}
2022
2023	return VINF_SUCCESS;
2024	}
2025
2026
2027	/**
2028	* Does per-VM VT-x termination.
2029	*
2030	* @returns VBox status code.
2031	* @param pVM Pointer to the VM.
2032	*/
2033	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2034	{
2035	LogFlowFunc(("pVM=%p\n", pVM));
2036
2037	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2038	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2039	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2040	#endif
2041	hmR0VmxStructsFree(pVM);
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Sets up the VM for execution under VT-x.
2048	* This function is only called once per-VM during initialization.
2049	*
2050	* @returns VBox status code.
2051	* @param pVM Pointer to the VM.
2052	*/
2053	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2054	{
2055	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2056	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2057
2058	LogFlowFunc(("pVM=%p\n", pVM));
2059
2060	/*
2061	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2062	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2063	*/
2064	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2065	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2066	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2067	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2068	{
2069	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2070	return VERR_INTERNAL_ERROR;
2071	}
2072
2073	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2074	/*
2075	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2076	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2077	*/
2078	if ( pVM->hm.s.fAllow64BitGuests
2079	&& !HMVMX_IS_64BIT_HOST_MODE())
2080	{
2081	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2082	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2083	}
2084	#endif
2085
2086	/* Initialize these always, see hmR3InitFinalizeR0().*/
2087	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2088	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2089
2090	/* Setup the tagged-TLB flush handlers. */
2091	int rc = hmR0VmxSetupTaggedTlb(pVM);
2092	if (RT_FAILURE(rc))
2093	{
2094	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2095	return rc;
2096	}
2097
2098	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2099	{
2100	PVMCPU pVCpu = &pVM->aCpus[i];
2101	AssertPtr(pVCpu);
2102	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2103
2104	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2105	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2106
2107	/* Set revision dword at the beginning of the VMCS structure. */
2108	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2109
2110	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2111	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2112	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2113	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2114
2115	/* Load this VMCS as the current VMCS. */
2116	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2117	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2118	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2119
2120	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2121	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2122	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2123
2124	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2125	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2126	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2127
2128	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2129	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2130	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2131
2132	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2133	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2134	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2135
2136	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2137	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2138	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2139
2140	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2141	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2142	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2143	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2144	#endif
2145
2146	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2147	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2148	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2149	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2150
2151	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2152	}
2153
2154	return VINF_SUCCESS;
2155	}
2156
2157
2158	/**
2159	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2160	* the VMCS.
2161	*
2162	* @returns VBox status code.
2163	* @param pVM Pointer to the VM.
2164	* @param pVCpu Pointer to the VMCPU.
2165	*/
2166	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2167	{
2168	RTCCUINTREG uReg = ASMGetCR0();
2169	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2170	AssertRCReturn(rc, rc);
2171
2172	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2173	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2174	if (HMVMX_IS_64BIT_HOST_MODE())
2175	{
2176	uint64_t uRegCR3 = HMR0Get64bitCR3();
2177	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2178	}
2179	else
2180	#endif
2181	{
2182	uReg = ASMGetCR3();
2183	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2184	}
2185	AssertRCReturn(rc, rc);
2186
2187	uReg = ASMGetCR4();
2188	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2189	AssertRCReturn(rc, rc);
2190	return rc;
2191	}
2192
2193
2194	/**
2195	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2196	* the host-state area in the VMCS.
2197	*
2198	* @returns VBox status code.
2199	* @param pVM Pointer to the VM.
2200	* @param pVCpu Pointer to the VMCPU.
2201	*/
2202	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2203	{
2204	int rc = VERR_INTERNAL_ERROR_5;
2205	RTSEL uSelDS = 0;
2206	RTSEL uSelES = 0;
2207	RTSEL uSelFS = 0;
2208	RTSEL uSelGS = 0;
2209	RTSEL uSelTR = 0;
2210
2211	/*
2212	* Host DS, ES, FS and GS segment registers.
2213	*/
2214	#if HC_ARCH_BITS == 64
2215	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2216	uSelDS = ASMGetDS();
2217	uSelES = ASMGetES();
2218	uSelFS = ASMGetFS();
2219	uSelGS = ASMGetGS();
2220	#endif
2221
2222	/*
2223	* Host CS and SS segment registers.
2224	*/
2225	RTSEL uSelCS;
2226	RTSEL uSelSS;
2227	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2228	if (HMVMX_IS_64BIT_HOST_MODE())
2229	{
2230	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2231	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2232	}
2233	else
2234	{
2235	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2236	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2237	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2238	}
2239	#else
2240	uSelCS = ASMGetCS();
2241	uSelSS = ASMGetSS();
2242	#endif
2243
2244	/*
2245	* Host TR segment register.
2246	*/
2247	uSelTR = ASMGetTR();
2248
2249	#if HC_ARCH_BITS == 64
2250	/*
2251	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2252	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2253	*/
2254	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2255	{
2256	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2257	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2258	uSelDS = 0;
2259	}
2260	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2261	{
2262	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2263	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2264	uSelES = 0;
2265	}
2266	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2267	{
2268	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2269	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2270	uSelFS = 0;
2271	}
2272	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2273	{
2274	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2275	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2276	uSelGS = 0;
2277	}
2278	#endif
2279
2280	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2281	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2282	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2283	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2284	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2285	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2286	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2287	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2288	Assert(uSelCS);
2289	Assert(uSelTR);
2290
2291	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2292	#if 0
2293	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2294	Assert(uSelSS != 0);
2295	#endif
2296
2297	/* Write these host selector fields into the host-state area in the VMCS. */
2298	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2299	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2300	#if HC_ARCH_BITS == 64
2301	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2302	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2303	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2304	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2305	#endif
2306	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2307
2308	/*
2309	* Host GDTR and IDTR.
2310	*/
2311	RTGDTR Gdtr;
2312	RT_ZERO(Gdtr);
2313	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2314	if (HMVMX_IS_64BIT_HOST_MODE())
2315	{
2316	X86XDTR64 Gdtr64;
2317	X86XDTR64 Idtr64;
2318	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2319	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2320	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2321
2322	Gdtr.cbGdt = Gdtr64.cb;
2323	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2324	}
2325	else
2326	#endif
2327	{
2328	RTIDTR Idtr;
2329	ASMGetGDTR(&Gdtr);
2330	ASMGetIDTR(&Idtr);
2331	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2332	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2333
2334	#if HC_ARCH_BITS == 64
2335	/*
2336	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2337	* maximum limit (0xffff) on every VM-exit.
2338	*/
2339	if (Gdtr.cbGdt != 0xffff)
2340	{
2341	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2342	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2343	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2344	}
2345
2346	/*
2347	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2348	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2349	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2350	*/
2351	if (Idtr.cbIdt < 0x0fff)
2352	{
2353	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2354	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2355	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2356	}
2357	#endif
2358	}
2359
2360	/*
2361	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2362	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2363	*/
2364	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2365	{
2366	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2367	return VERR_VMX_INVALID_HOST_STATE;
2368	}
2369
2370	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2371	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2372	if (HMVMX_IS_64BIT_HOST_MODE())
2373	{
2374	/* We need the 64-bit TR base for hybrid darwin. */
2375	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2376	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2377	}
2378	else
2379	#endif
2380	{
2381	uintptr_t uTRBase;
2382	#if HC_ARCH_BITS == 64
2383	uTRBase = X86DESC64_BASE(pDesc);
2384
2385	/*
2386	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2387	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2388	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2389	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2390	*
2391	* [1] See Intel spec. 3.5 "System Descriptor Types".
2392	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2393	*/
2394	Assert(pDesc->System.u4Type == 11);
2395	if ( pDesc->System.u16LimitLow != 0x67
2396	\|\| pDesc->System.u4LimitHigh)
2397	{
2398	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2399	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2400
2401	/* Store the GDTR here as we need it while restoring TR. */
2402	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2403	}
2404	#else
2405	uTRBase = X86DESC_BASE(pDesc);
2406	#endif
2407	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2408	}
2409	AssertRCReturn(rc, rc);
2410
2411	/*
2412	* Host FS base and GS base.
2413	*/
2414	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2415	if (HMVMX_IS_64BIT_HOST_MODE())
2416	{
2417	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2418	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2419	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2420	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2421
2422	# if HC_ARCH_BITS == 64
2423	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2424	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2425	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2426	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2427	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2428	# endif
2429	}
2430	#endif
2431	return rc;
2432	}
2433
2434
2435	/**
2436	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2437	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2438	* the host after every successful VM exit.
2439	*
2440	* @returns VBox status code.
2441	* @param pVM Pointer to the VM.
2442	* @param pVCpu Pointer to the VMCPU.
2443	*/
2444	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2445	{
2446	AssertPtr(pVCpu);
2447	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2448
2449	int rc = VINF_SUCCESS;
2450	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2451	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2452	uint32_t cHostMsrs = 0;
2453	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2454
2455	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2456	{
2457	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2458
2459	# if HC_ARCH_BITS == 64
2460	/* Paranoia. 64-bit code requires these bits to be set always. */
2461	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2462
2463	/*
2464	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2465	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2466	* some reason (e.g. allow transparent reads) we would activate the code below.
2467	*/
2468	# if 0
2469	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2470	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2471	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2472	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2473	if (CPUMIsGuestInLongMode(pVCpu))
2474	{
2475	uint64_t u64GuestEfer;
2476	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2477	AssertRC(rc);
2478
2479	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2480	{
2481	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2482	pHostMsr->u32Reserved = 0;
2483	pHostMsr->u64Value = u64HostEfer;
2484	pHostMsr++; cHostMsrs++;
2485	}
2486	}
2487	# endif
2488	# else /* HC_ARCH_BITS != 64 */
2489	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2490	pHostMsr->u32Reserved = 0;
2491	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2492	if (CPUMIsGuestInLongMode(pVCpu))
2493	{
2494	/* Must match the EFER value in our 64 bits switcher. */
2495	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2496	}
2497	else
2498	# endif
2499	pHostMsr->u64Value = u64HostEfer;
2500	pHostMsr++; cHostMsrs++;
2501	# endif /* HC_ARCH_BITS == 64 */
2502	}
2503
2504	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2505	if (HMVMX_IS_64BIT_HOST_MODE())
2506	{
2507	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2508	pHostMsr->u32Reserved = 0;
2509	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2510	pHostMsr++; cHostMsrs++;
2511	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2512	pHostMsr->u32Reserved = 0;
2513	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2514	pHostMsr++; cHostMsrs++;
2515	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2516	pHostMsr->u32Reserved = 0;
2517	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2518	pHostMsr++; cHostMsrs++;
2519	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2520	pHostMsr->u32Reserved = 0;
2521	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2522	pHostMsr++; cHostMsrs++;
2523	}
2524	# endif
2525
2526	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2527	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2528	{
2529	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2530	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2531	}
2532
2533	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2534	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2535
2536	/*
2537	* Host Sysenter MSRs.
2538	*/
2539	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2540	AssertRCReturn(rc, rc);
2541	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2542	if (HMVMX_IS_64BIT_HOST_MODE())
2543	{
2544	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2545	AssertRCReturn(rc, rc);
2546	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2547	}
2548	else
2549	{
2550	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2551	AssertRCReturn(rc, rc);
2552	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2553	}
2554	#elif HC_ARCH_BITS == 32
2555	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2556	AssertRCReturn(rc, rc);
2557	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2558	#else
2559	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2560	AssertRCReturn(rc, rc);
2561	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2562	#endif
2563	AssertRCReturn(rc, rc);
2564
2565	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2566	* hmR0VmxSetupExitCtls() !! */
2567	return rc;
2568	}
2569
2570
2571	/**
2572	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2573	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2574	* controls".
2575	*
2576	* @returns VBox status code.
2577	* @param pVCpu Pointer to the VMCPU.
2578	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2579	* out-of-sync. Make sure to update the required fields
2580	* before using them.
2581	*
2582	* @remarks No-long-jump zone!!!
2583	*/
2584	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2585	{
2586	int rc = VINF_SUCCESS;
2587	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2588	{
2589	PVM pVM = pVCpu->CTX_SUFF(pVM);
2590	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2591	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2592
2593	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2594	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2595
2596	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2597	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2598	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2599	else
2600	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2601
2602	/*
2603	* The following should not be set (since we're not in SMM mode):
2604	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2605	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2606	*/
2607
2608	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2609	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2610	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2611
2612	if ((val & zap) != val)
2613	{
2614	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2615	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2616	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2617	}
2618
2619	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2620	AssertRCReturn(rc, rc);
2621
2622	/* Update VCPU with the currently set VM-exit controls. */
2623	pVCpu->hm.s.vmx.u32EntryCtls = val;
2624	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2625	}
2626	return rc;
2627	}
2628
2629
2630	/**
2631	* Sets up the VM-exit controls in the VMCS.
2632	*
2633	* @returns VBox status code.
2634	* @param pVM Pointer to the VM.
2635	* @param pVCpu Pointer to the VMCPU.
2636	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2637	* out-of-sync. Make sure to update the required fields
2638	* before using them.
2639	*
2640	* @remarks requires EFER.
2641	*/
2642	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2643	{
2644	int rc = VINF_SUCCESS;
2645	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2646	{
2647	PVM pVM = pVCpu->CTX_SUFF(pVM);
2648	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2649	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2650
2651	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2652	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2653
2654	/*
2655	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2656	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2657	*/
2658	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2659	if (HMVMX_IS_64BIT_HOST_MODE())
2660	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2661	else
2662	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2663	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2664	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2665	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2666	else
2667	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2668	#endif
2669
2670	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2671	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2672
2673	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2674	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2675	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2676	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2677	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2678
2679	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2680	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2681
2682	if ((val & zap) != val)
2683	{
2684	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2685	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2686	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2687	}
2688
2689	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2690	AssertRCReturn(rc, rc);
2691
2692	/* Update VCPU with the currently set VM-exit controls. */
2693	pVCpu->hm.s.vmx.u32ExitCtls = val;
2694	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2695	}
2696	return rc;
2697	}
2698
2699
2700	/**
2701	* Loads the guest APIC and related state.
2702	*
2703	* @returns VBox status code.
2704	* @param pVM Pointer to the VM.
2705	* @param pVCpu Pointer to the VMCPU.
2706	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2707	* out-of-sync. Make sure to update the required fields
2708	* before using them.
2709	*/
2710	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2711	{
2712	int rc = VINF_SUCCESS;
2713	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2714	{
2715	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2716	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2717	{
2718	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2719
2720	bool fPendingIntr = false;
2721	uint8_t u8Tpr = 0;
2722	uint8_t u8PendingIntr = 0;
2723	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2724	AssertRCReturn(rc, rc);
2725
2726	/*
2727	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2728	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2729	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2730	* the interrupt when we VM-exit for other reasons.
2731	*/
2732	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2733	uint32_t u32TprThreshold = 0;
2734	if (fPendingIntr)
2735	{
2736	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2737	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2738	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2739	if (u8PendingPriority <= u8TprPriority)
2740	u32TprThreshold = u8PendingPriority;
2741	else
2742	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2743	}
2744	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2745
2746	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2747	AssertRCReturn(rc, rc);
2748	}
2749
2750	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2751	}
2752	return rc;
2753	}
2754
2755
2756	/**
2757	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2758	*
2759	* @returns Guest's interruptibility-state.
2760	* @param pVCpu Pointer to the VMCPU.
2761	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2762	* out-of-sync. Make sure to update the required fields
2763	* before using them.
2764	*
2765	* @remarks No-long-jump zone!!!
2766	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2767	*/
2768	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2769	{
2770	/*
2771	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2772	* inhibit interrupts or clear any existing interrupt-inhibition.
2773	*/
2774	uint32_t uIntrState = 0;
2775	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2776	{
2777	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2778	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2779	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2780	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2781	{
2782	/*
2783	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2784	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2785	*/
2786	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2787	}
2788	else if (pMixedCtx->eflags.Bits.u1IF)
2789	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2790	else
2791	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2792	}
2793	return uIntrState;
2794	}
2795
2796
2797	/**
2798	* Loads the guest's interruptibility-state into the guest-state area in the
2799	* VMCS.
2800	*
2801	* @returns VBox status code.
2802	* @param pVCpu Pointer to the VMCPU.
2803	* @param uIntrState The interruptibility-state to set.
2804	*/
2805	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2806	{
2807	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2808	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2809	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2810	AssertRCReturn(rc, rc);
2811	return rc;
2812	}
2813
2814
2815	/**
2816	* Loads the guest's RIP into the guest-state area in the VMCS.
2817	*
2818	* @returns VBox status code.
2819	* @param pVCpu Pointer to the VMCPU.
2820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2821	* out-of-sync. Make sure to update the required fields
2822	* before using them.
2823	*
2824	* @remarks No-long-jump zone!!!
2825	*/
2826	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2827	{
2828	int rc = VINF_SUCCESS;
2829	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2830	{
2831	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2832	AssertRCReturn(rc, rc);
2833	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2834	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2835	}
2836	return rc;
2837	}
2838
2839
2840	/**
2841	* Loads the guest's RSP into the guest-state area in the VMCS.
2842	*
2843	* @returns VBox status code.
2844	* @param pVCpu Pointer to the VMCPU.
2845	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2846	* out-of-sync. Make sure to update the required fields
2847	* before using them.
2848	*
2849	* @remarks No-long-jump zone!!!
2850	*/
2851	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2852	{
2853	int rc = VINF_SUCCESS;
2854	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2855	{
2856	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2857	AssertRCReturn(rc, rc);
2858	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2859	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2860	}
2861	return rc;
2862	}
2863
2864
2865	/**
2866	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2867	*
2868	* @returns VBox status code.
2869	* @param pVCpu Pointer to the VMCPU.
2870	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2871	* out-of-sync. Make sure to update the required fields
2872	* before using them.
2873	*
2874	* @remarks No-long-jump zone!!!
2875	*/
2876	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2877	{
2878	int rc = VINF_SUCCESS;
2879	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2880	{
2881	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2882	Let us assert it as such and use 32-bit VMWRITE. */
2883	Assert(!(pMixedCtx->rflags.u64 >> 32));
2884	X86EFLAGS uEFlags = pMixedCtx->eflags;
2885	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2886	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2887
2888	/*
2889	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2890	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2891	*/
2892	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2893	{
2894	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2895	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2896	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2897	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2898	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2899	}
2900
2901	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2902	AssertRCReturn(rc, rc);
2903
2904	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2905	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2906	}
2907	return rc;
2908	}
2909
2910
2911	/**
2912	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2913	*
2914	* @returns VBox status code.
2915	* @param pVCpu Pointer to the VMCPU.
2916	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2917	* out-of-sync. Make sure to update the required fields
2918	* before using them.
2919	*
2920	* @remarks No-long-jump zone!!!
2921	*/
2922	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2923	{
2924	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2925	AssertRCReturn(rc, rc);
2926	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2927	AssertRCReturn(rc, rc);
2928	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2929	AssertRCReturn(rc, rc);
2930	return rc;
2931	}
2932
2933
2934	/**
2935	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2936	* in the VMCS.
2937	*
2938	* @returns VBox status code.
2939	* @param pVM Pointer to the VM.
2940	* @param pVCpu Pointer to the VMCPU.
2941	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2942	* out-of-sync. Make sure to update the required fields
2943	* before using them.
2944	*
2945	* @remarks No-long-jump zone!!!
2946	*/
2947	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2948	{
2949	int rc = VINF_SUCCESS;
2950	PVM pVM = pVCpu->CTX_SUFF(pVM);
2951
2952	/*
2953	* Guest CR0.
2954	* Guest FPU.
2955	*/
2956	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2957	{
2958	Assert(!(pCtx->cr0 >> 32));
2959	uint32_t u32GuestCR0 = pCtx->cr0;
2960
2961	/* The guest's view (read access) of its CR0 is unblemished. */
2962	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2963	AssertRCReturn(rc, rc);
2964	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2965
2966	/* Setup VT-x's view of the guest CR0. */
2967	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2968	if (pVM->hm.s.fNestedPaging)
2969	{
2970	if (CPUMIsGuestPagingEnabledEx(pCtx))
2971	{
2972	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2973	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2974	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2975	}
2976	else
2977	{
2978	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2979	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2980	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2981	}
2982
2983	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2984	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2985	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2986
2987	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2988	AssertRCReturn(rc, rc);
2989	}
2990	else
2991	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2992
2993	/*
2994	* Guest FPU bits.
2995	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2996	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2997	*/
2998	u32GuestCR0 \|= X86_CR0_NE;
2999	bool fInterceptNM = false;
3000	if (CPUMIsGuestFPUStateActive(pVCpu))
3001	{
3002	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3003	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3004	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3005	}
3006	else
3007	{
3008	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3009	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3010	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3011	}
3012
3013	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3014	bool fInterceptMF = false;
3015	if (!(pCtx->cr0 & X86_CR0_NE))
3016	fInterceptMF = true;
3017
3018	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3019	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3020	{
3021	Assert(PDMVmmDevHeapIsEnabled(pVM));
3022	Assert(pVM->hm.s.vmx.pRealModeTSS);
3023	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3024	fInterceptNM = true;
3025	fInterceptMF = true;
3026	}
3027	else
3028	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3029
3030	if (fInterceptNM)
3031	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3032	else
3033	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3034
3035	if (fInterceptMF)
3036	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3037	else
3038	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3039
3040	/* Additional intercepts for debugging, define these yourself explicitly. */
3041	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3042	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3043	\| RT_BIT(X86_XCPT_BP)
3044	\| RT_BIT(X86_XCPT_DB)
3045	\| RT_BIT(X86_XCPT_DE)
3046	\| RT_BIT(X86_XCPT_NM)
3047	\| RT_BIT(X86_XCPT_UD)
3048	\| RT_BIT(X86_XCPT_NP)
3049	\| RT_BIT(X86_XCPT_SS)
3050	\| RT_BIT(X86_XCPT_GP)
3051	\| RT_BIT(X86_XCPT_PF)
3052	\| RT_BIT(X86_XCPT_MF)
3053	;
3054	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3055	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3056	#endif
3057
3058	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3059
3060	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3061	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3062	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3063	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3064	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3065	else
3066	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3067
3068	u32GuestCR0 \|= uSetCR0;
3069	u32GuestCR0 &= uZapCR0;
3070	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3071
3072	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3073	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3074	AssertRCReturn(rc, rc);
3075	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3076	AssertRCReturn(rc, rc);
3077	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3078
3079	/*
3080	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3081	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3082	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3083	*/
3084	uint32_t u32CR0Mask = 0;
3085	u32CR0Mask = X86_CR0_PE
3086	\| X86_CR0_NE
3087	\| X86_CR0_WP
3088	\| X86_CR0_PG
3089	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3090	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3091	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3092	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3093	u32CR0Mask &= ~X86_CR0_PE;
3094	if (pVM->hm.s.fNestedPaging)
3095	u32CR0Mask &= ~X86_CR0_WP;
3096
3097	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3098	if (fInterceptNM)
3099	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3100	else
3101	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3102
3103	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3104	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3105	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3106	AssertRCReturn(rc, rc);
3107
3108	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3109	}
3110
3111	/*
3112	* Guest CR2.
3113	* It's always loaded in the assembler code. Nothing to do here.
3114	*/
3115
3116	/*
3117	* Guest CR3.
3118	*/
3119	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3120	{
3121	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3122	if (pVM->hm.s.fNestedPaging)
3123	{
3124	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3125
3126	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3127	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3128	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3129	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3130
3131	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3132	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3133	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3134
3135	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3136	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3137	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3138	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3139
3140	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3141	AssertRCReturn(rc, rc);
3142	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3143
3144	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3145	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3146	{
3147	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3148	if (CPUMIsGuestInPAEModeEx(pCtx))
3149	{
3150	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3151	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3152	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3153	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3154	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3155	}
3156
3157	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3158	have Unrestricted Execution to handle the guest when it's not using paging. */
3159	GCPhysGuestCR3 = pCtx->cr3;
3160	}
3161	else
3162	{
3163	/*
3164	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3165	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3166	* EPT takes care of translating it to host-physical addresses.
3167	*/
3168	RTGCPHYS GCPhys;
3169	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3170	Assert(PDMVmmDevHeapIsEnabled(pVM));
3171
3172	/* We obtain it here every time as the guest could have relocated this PCI region. */
3173	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3174	AssertRCReturn(rc, rc);
3175
3176	GCPhysGuestCR3 = GCPhys;
3177	}
3178
3179	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3180	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3181	}
3182	else
3183	{
3184	/* Non-nested paging case, just use the hypervisor's CR3. */
3185	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3186
3187	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3188	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3189	}
3190	AssertRCReturn(rc, rc);
3191
3192	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3193	}
3194
3195	/*
3196	* Guest CR4.
3197	*/
3198	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3199	{
3200	Assert(!(pCtx->cr4 >> 32));
3201	uint32_t u32GuestCR4 = pCtx->cr4;
3202
3203	/* The guest's view of its CR4 is unblemished. */
3204	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3205	AssertRCReturn(rc, rc);
3206	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3207
3208	/* Setup VT-x's view of the guest CR4. */
3209	/*
3210	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3211	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3212	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3213	*/
3214	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3215	{
3216	Assert(pVM->hm.s.vmx.pRealModeTSS);
3217	Assert(PDMVmmDevHeapIsEnabled(pVM));
3218	u32GuestCR4 &= ~X86_CR4_VME;
3219	}
3220
3221	if (pVM->hm.s.fNestedPaging)
3222	{
3223	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3224	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3225	{
3226	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3227	u32GuestCR4 \|= X86_CR4_PSE;
3228	/* Our identity mapping is a 32 bits page directory. */
3229	u32GuestCR4 &= ~X86_CR4_PAE;
3230	}
3231	/* else use guest CR4.*/
3232	}
3233	else
3234	{
3235	/*
3236	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3237	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3238	*/
3239	switch (pVCpu->hm.s.enmShadowMode)
3240	{
3241	case PGMMODE_REAL: /* Real-mode. */
3242	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3243	case PGMMODE_32_BIT: /* 32-bit paging. */
3244	{
3245	u32GuestCR4 &= ~X86_CR4_PAE;
3246	break;
3247	}
3248
3249	case PGMMODE_PAE: /* PAE paging. */
3250	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3251	{
3252	u32GuestCR4 \|= X86_CR4_PAE;
3253	break;
3254	}
3255
3256	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3257	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3258	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3259	break;
3260	#endif
3261	default:
3262	AssertFailed();
3263	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3264	}
3265	}
3266
3267	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3268	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3269	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3270	u32GuestCR4 \|= uSetCR4;
3271	u32GuestCR4 &= uZapCR4;
3272
3273	/* Write VT-x's view of the guest CR4 into the VMCS. */
3274	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3275	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3276	AssertRCReturn(rc, rc);
3277
3278	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3279	uint32_t u32CR4Mask = 0;
3280	u32CR4Mask = X86_CR4_VME
3281	\| X86_CR4_PAE
3282	\| X86_CR4_PGE
3283	\| X86_CR4_PSE
3284	\| X86_CR4_VMXE;
3285	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3286	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3287	AssertRCReturn(rc, rc);
3288
3289	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3290	}
3291	return rc;
3292	}
3293
3294
3295	/**
3296	* Loads the guest debug registers into the guest-state area in the VMCS.
3297	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3298	*
3299	* @returns VBox status code.
3300	* @param pVCpu Pointer to the VMCPU.
3301	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3302	* out-of-sync. Make sure to update the required fields
3303	* before using them.
3304	*
3305	* @remarks No-long-jump zone!!!
3306	*/
3307	static int hmR0VmxLoadGuestDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3308	{
3309	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3310	return VINF_SUCCESS;
3311
3312	#ifdef VBOX_STRICT
3313	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3314	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3315	{
3316	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3317	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3318	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3319	}
3320	#endif
3321
3322	int rc;
3323	PVM pVM = pVCpu->CTX_SUFF(pVM);
3324	bool fInterceptDB = false;
3325	bool fInterceptMovDRx = false;
3326	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3327	{
3328	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3329	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3330	{
3331	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3332	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3333	AssertRCReturn(rc, rc);
3334	Assert(fInterceptDB == false);
3335	}
3336	else
3337	{
3338	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3339	pVCpu->hm.s.fClearTrapFlag = true;
3340	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3341	fInterceptDB = true;
3342	}
3343	}
3344
3345	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3346	{
3347	/*
3348	* Use the combined guest and host DRx values found in the hypervisor
3349	* register set because the debugger has breakpoints active or someone
3350	* is single stepping on the host side without a monitor trap flag.
3351	*
3352	* Note! DBGF expects a clean DR6 state before executing guest code.
3353	*/
3354	if (!CPUMIsHyperDebugStateActive(pVCpu))
3355	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3356	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3357	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3358
3359	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3360	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3361	AssertRCReturn(rc, rc);
3362
3363	fInterceptDB = true;
3364	fInterceptMovDRx = true;
3365	}
3366	else
3367	{
3368	/*
3369	* If the guest has enabled debug registers, we need to load them prior to
3370	* executing guest code so they'll trigger at the right time.
3371	*/
3372	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3373	{
3374	if (!CPUMIsGuestDebugStateActive(pVCpu))
3375	{
3376	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3377	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3378	}
3379	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3380	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3381	}
3382	/*
3383	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3384	* must intercept #DB in order to maintain a correct DR6 guest value.
3385	*/
3386	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3387	{
3388	fInterceptMovDRx = true;
3389	fInterceptDB = true;
3390	}
3391
3392	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3393	AssertRCReturn(rc, rc);
3394	}
3395
3396	/*
3397	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3398	*/
3399	if (fInterceptDB)
3400	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3401	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3402	{
3403	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3404	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3405	#endif
3406	}
3407	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3408	AssertRCReturn(rc, rc);
3409
3410	/*
3411	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3412	*/
3413	if (fInterceptMovDRx)
3414	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3415	else
3416	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3417	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3418	AssertRCReturn(rc, rc);
3419
3420	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3421	return VINF_SUCCESS;
3422	}
3423
3424
3425	#ifdef VBOX_STRICT
3426	/**
3427	* Strict function to validate segment registers.
3428	*
3429	* @remarks Requires CR0.
3430	*/
3431	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3432	{
3433	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3434	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3435	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3436	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3437	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3438	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3439	{
3440	/* Protected mode checks */
3441	/* CS */
3442	Assert(pCtx->cs.Attr.n.u1Present);
3443	Assert(!(pCtx->cs.Attr.u & 0xf00));
3444	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3445	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3446	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3447	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3448	\|\| (pCtx->cs.Attr.n.u1Granularity));
3449	/* CS cannot be loaded with NULL in protected mode. */
3450	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3451	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3452	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3453	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3454	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3455	else
3456	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3457	/* SS */
3458	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3459	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3460	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3461	if ( !(pCtx->cr0 & X86_CR0_PE)
3462	\|\| pCtx->cs.Attr.n.u4Type == 3)
3463	{
3464	Assert(!pCtx->ss.Attr.n.u2Dpl);
3465	}
3466	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3467	{
3468	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3469	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3470	Assert(pCtx->ss.Attr.n.u1Present);
3471	Assert(!(pCtx->ss.Attr.u & 0xf00));
3472	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3473	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3474	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3475	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3476	\|\| (pCtx->ss.Attr.n.u1Granularity));
3477	}
3478	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3479	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3480	{
3481	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3482	Assert(pCtx->ds.Attr.n.u1Present);
3483	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3484	Assert(!(pCtx->ds.Attr.u & 0xf00));
3485	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3486	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3487	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3488	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3489	\|\| (pCtx->ds.Attr.n.u1Granularity));
3490	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3491	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3492	}
3493	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3494	{
3495	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3496	Assert(pCtx->es.Attr.n.u1Present);
3497	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3498	Assert(!(pCtx->es.Attr.u & 0xf00));
3499	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3500	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3501	\|\| !(pCtx->es.Attr.n.u1Granularity));
3502	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3503	\|\| (pCtx->es.Attr.n.u1Granularity));
3504	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3505	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3506	}
3507	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3508	{
3509	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3510	Assert(pCtx->fs.Attr.n.u1Present);
3511	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3512	Assert(!(pCtx->fs.Attr.u & 0xf00));
3513	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3514	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3515	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3516	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3517	\|\| (pCtx->fs.Attr.n.u1Granularity));
3518	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3519	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3520	}
3521	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3522	{
3523	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3524	Assert(pCtx->gs.Attr.n.u1Present);
3525	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3526	Assert(!(pCtx->gs.Attr.u & 0xf00));
3527	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3528	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3529	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3530	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3531	\|\| (pCtx->gs.Attr.n.u1Granularity));
3532	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3533	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3534	}
3535	/* 64-bit capable CPUs. */
3536	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3537	Assert(!(pCtx->cs.u64Base >> 32));
3538	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3539	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3540	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3541	# endif
3542	}
3543	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3544	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3545	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3546	{
3547	/* Real and v86 mode checks. */
3548	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3549	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3550	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3551	{
3552	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3553	}
3554	else
3555	{
3556	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3557	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3558	}
3559
3560	/* CS */
3561	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3562	Assert(pCtx->cs.u32Limit == 0xffff);
3563	Assert(u32CSAttr == 0xf3);
3564	/* SS */
3565	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3566	Assert(pCtx->ss.u32Limit == 0xffff);
3567	Assert(u32SSAttr == 0xf3);
3568	/* DS */
3569	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3570	Assert(pCtx->ds.u32Limit == 0xffff);
3571	Assert(u32DSAttr == 0xf3);
3572	/* ES */
3573	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3574	Assert(pCtx->es.u32Limit == 0xffff);
3575	Assert(u32ESAttr == 0xf3);
3576	/* FS */
3577	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3578	Assert(pCtx->fs.u32Limit == 0xffff);
3579	Assert(u32FSAttr == 0xf3);
3580	/* GS */
3581	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3582	Assert(pCtx->gs.u32Limit == 0xffff);
3583	Assert(u32GSAttr == 0xf3);
3584	/* 64-bit capable CPUs. */
3585	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3586	Assert(!(pCtx->cs.u64Base >> 32));
3587	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3588	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3589	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3590	# endif
3591	}
3592	}
3593	#endif /* VBOX_STRICT */
3594
3595
3596	/**
3597	* Writes a guest segment register into the guest-state area in the VMCS.
3598	*
3599	* @returns VBox status code.
3600	* @param pVCpu Pointer to the VMCPU.
3601	* @param idxSel Index of the selector in the VMCS.
3602	* @param idxLimit Index of the segment limit in the VMCS.
3603	* @param idxBase Index of the segment base in the VMCS.
3604	* @param idxAccess Index of the access rights of the segment in the VMCS.
3605	* @param pSelReg Pointer to the segment selector.
3606	* @param pCtx Pointer to the guest-CPU context.
3607	*
3608	* @remarks No-long-jump zone!!!
3609	*/
3610	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3611	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3612	{
3613	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3614	AssertRCReturn(rc, rc);
3615	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3616	AssertRCReturn(rc, rc);
3617	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3618	AssertRCReturn(rc, rc);
3619
3620	uint32_t u32Access = pSelReg->Attr.u;
3621	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3622	{
3623	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3624	u32Access = 0xf3;
3625	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3626	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3627	}
3628	else
3629	{
3630	/*
3631	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3632	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3633	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3634	* loaded in protected-mode have their attribute as 0.
3635	*/
3636	if (!u32Access)
3637	u32Access = X86DESCATTR_UNUSABLE;
3638	}
3639
3640	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3641	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3642	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3643
3644	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3645	AssertRCReturn(rc, rc);
3646	return rc;
3647	}
3648
3649
3650	/**
3651	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3652	* into the guest-state area in the VMCS.
3653	*
3654	* @returns VBox status code.
3655	* @param pVM Pointer to the VM.
3656	* @param pVCPU Pointer to the VMCPU.
3657	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3658	* out-of-sync. Make sure to update the required fields
3659	* before using them.
3660	*
3661	* @remarks Requires CR0 (strict builds validation).
3662	* @remarks No-long-jump zone!!!
3663	*/
3664	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3665	{
3666	int rc = VERR_INTERNAL_ERROR_5;
3667	PVM pVM = pVCpu->CTX_SUFF(pVM);
3668
3669	/*
3670	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3671	*/
3672	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3673	{
3674	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3675	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3676	{
3677	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3678	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3679	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3680	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3681	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3682	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3683	}
3684
3685	#ifdef VBOX_WITH_REM
3686	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3687	{
3688	Assert(pVM->hm.s.vmx.pRealModeTSS);
3689	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3690	if ( pVCpu->hm.s.vmx.fWasInRealMode
3691	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3692	{
3693	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3694	in real-mode (e.g. OpenBSD 4.0) */
3695	REMFlushTBs(pVM);
3696	Log4(("Load: Switch to protected mode detected!\n"));
3697	pVCpu->hm.s.vmx.fWasInRealMode = false;
3698	}
3699	}
3700	#endif
3701	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3702	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3703	AssertRCReturn(rc, rc);
3704	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3705	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3706	AssertRCReturn(rc, rc);
3707	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3708	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3709	AssertRCReturn(rc, rc);
3710	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3711	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3712	AssertRCReturn(rc, rc);
3713	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3714	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3715	AssertRCReturn(rc, rc);
3716	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3717	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3718	AssertRCReturn(rc, rc);
3719
3720	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3721	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3722	#ifdef VBOX_STRICT
3723	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3724	#endif
3725	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3726	}
3727
3728	/*
3729	* Guest TR.
3730	*/
3731	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3732	{
3733	/*
3734	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3735	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3736	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3737	*/
3738	uint16_t u16Sel = 0;
3739	uint32_t u32Limit = 0;
3740	uint64_t u64Base = 0;
3741	uint32_t u32AccessRights = 0;
3742
3743	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3744	{
3745	u16Sel = pMixedCtx->tr.Sel;
3746	u32Limit = pMixedCtx->tr.u32Limit;
3747	u64Base = pMixedCtx->tr.u64Base;
3748	u32AccessRights = pMixedCtx->tr.Attr.u;
3749	}
3750	else
3751	{
3752	Assert(pVM->hm.s.vmx.pRealModeTSS);
3753	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3754
3755	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3756	RTGCPHYS GCPhys;
3757	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3758	AssertRCReturn(rc, rc);
3759
3760	X86DESCATTR DescAttr;
3761	DescAttr.u = 0;
3762	DescAttr.n.u1Present = 1;
3763	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3764
3765	u16Sel = 0;
3766	u32Limit = HM_VTX_TSS_SIZE;
3767	u64Base = GCPhys; /* in real-mode phys = virt. */
3768	u32AccessRights = DescAttr.u;
3769	}
3770
3771	/* Validate. */
3772	Assert(!(u16Sel & RT_BIT(2)));
3773	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3774	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3775	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3776	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3777	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3778	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3779	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3780	Assert( (u32Limit & 0xfff) == 0xfff
3781	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3782	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3783	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3784
3785	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3786	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3787	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3788	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3789
3790	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3791	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3792	}
3793
3794	/*
3795	* Guest GDTR.
3796	*/
3797	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3798	{
3799	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3800	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3801
3802	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3803	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3804	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3805	}
3806
3807	/*
3808	* Guest LDTR.
3809	*/
3810	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3811	{
3812	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3813	uint32_t u32Access = 0;
3814	if (!pMixedCtx->ldtr.Attr.u)
3815	u32Access = X86DESCATTR_UNUSABLE;
3816	else
3817	u32Access = pMixedCtx->ldtr.Attr.u;
3818
3819	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3820	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3821	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3822	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3823
3824	/* Validate. */
3825	if (!(u32Access & X86DESCATTR_UNUSABLE))
3826	{
3827	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3828	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3829	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3830	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3831	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3832	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3833	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3834	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3835	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3836	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3837	}
3838
3839	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3840	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3841	}
3842
3843	/*
3844	* Guest IDTR.
3845	*/
3846	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3847	{
3848	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3849	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3850
3851	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3852	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3853	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3854	}
3855
3856	return VINF_SUCCESS;
3857	}
3858
3859
3860	/**
3861	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3862	* areas. These MSRs will automatically be loaded to the host CPU on every
3863	* successful VM entry and stored from the host CPU on every successful VM exit.
3864	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3865	*
3866	* @returns VBox status code.
3867	* @param pVCpu Pointer to the VMCPU.
3868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3869	* out-of-sync. Make sure to update the required fields
3870	* before using them.
3871	*
3872	* @remarks No-long-jump zone!!!
3873	*/
3874	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3875	{
3876	AssertPtr(pVCpu);
3877	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3878
3879	/*
3880	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3881	*/
3882	int rc = VINF_SUCCESS;
3883	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3884	{
3885	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3886	PVM pVM = pVCpu->CTX_SUFF(pVM);
3887	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3888	uint32_t cGuestMsrs = 0;
3889
3890	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3891	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3892	* when the guest really is in 64-bit mode. */
3893	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3894	if (fSupportsLongMode)
3895	{
3896	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3897	pGuestMsr->u32Reserved = 0;
3898	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3899	pGuestMsr++; cGuestMsrs++;
3900	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3901	pGuestMsr->u32Reserved = 0;
3902	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3903	pGuestMsr++; cGuestMsrs++;
3904	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3905	pGuestMsr->u32Reserved = 0;
3906	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3907	pGuestMsr++; cGuestMsrs++;
3908	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3909	pGuestMsr->u32Reserved = 0;
3910	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3911	pGuestMsr++; cGuestMsrs++;
3912	}
3913
3914	/*
3915	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3916	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3917	*/
3918	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3919	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3920	{
3921	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3922	pGuestMsr->u32Reserved = 0;
3923	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3924	AssertRCReturn(rc, rc);
3925	pGuestMsr++; cGuestMsrs++;
3926	}
3927
3928	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3929	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3930	{
3931	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3932	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3933	}
3934
3935	/* Update the VCPU's copy of the guest MSR count. */
3936	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3937	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3938	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3939	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3940
3941	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3942	}
3943
3944	/*
3945	* Guest Sysenter MSRs.
3946	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3947	* VM-exits on WRMSRs for these MSRs.
3948	*/
3949	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3950	{
3951	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3952	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3953	}
3954	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3955	{
3956	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3957	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3958	}
3959	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3960	{
3961	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3962	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3963	}
3964
3965	return rc;
3966	}
3967
3968
3969	/**
3970	* Loads the guest activity state into the guest-state area in the VMCS.
3971	*
3972	* @returns VBox status code.
3973	* @param pVCpu Pointer to the VMCPU.
3974	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3975	* out-of-sync. Make sure to update the required fields
3976	* before using them.
3977	*
3978	* @remarks No-long-jump zone!!!
3979	*/
3980	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3981	{
3982	/** @todo See if we can make use of other states, e.g.
3983	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3984	int rc = VINF_SUCCESS;
3985	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3986	{
3987	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3988	AssertRCReturn(rc, rc);
3989	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3990	}
3991	return rc;
3992	}
3993
3994
3995	/**
3996	* Sets up the appropriate function to run guest code.
3997	*
3998	* @returns VBox status code.
3999	* @param pVCpu Pointer to the VMCPU.
4000	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4001	* out-of-sync. Make sure to update the required fields
4002	* before using them.
4003	*
4004	* @remarks No-long-jump zone!!!
4005	*/
4006	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4007	{
4008	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4009	{
4010	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4011	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4012	#endif
4013	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4014	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4015	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4016	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4017	#else
4018	/* 64-bit host or hybrid host. */
4019	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4020	#endif
4021	}
4022	else
4023	{
4024	/* Guest is not in long mode, use the 32-bit handler. */
4025	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4026	}
4027	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4028	return VINF_SUCCESS;
4029	}
4030
4031
4032	/**
4033	* Wrapper for running the guest code in VT-x.
4034	*
4035	* @returns VBox strict status code.
4036	* @param pVM Pointer to the VM.
4037	* @param pVCpu Pointer to the VMCPU.
4038	* @param pCtx Pointer to the guest-CPU context.
4039	*
4040	* @remarks No-long-jump zone!!!
4041	*/
4042	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4043	{
4044	/*
4045	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4046	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4047	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4048	*/
4049	#ifdef VBOX_WITH_KERNEL_USING_XMM
4050	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4051	#else
4052	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4053	#endif
4054	}
4055
4056
4057	/**
4058	* Reports world-switch error and dumps some useful debug info.
4059	*
4060	* @param pVM Pointer to the VM.
4061	* @param pVCpu Pointer to the VMCPU.
4062	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4063	* @param pCtx Pointer to the guest-CPU context.
4064	* @param pVmxTransient Pointer to the VMX transient structure (only
4065	* exitReason updated).
4066	*/
4067	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4068	{
4069	Assert(pVM);
4070	Assert(pVCpu);
4071	Assert(pCtx);
4072	Assert(pVmxTransient);
4073	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4074
4075	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4076	switch (rcVMRun)
4077	{
4078	case VERR_VMX_INVALID_VMXON_PTR:
4079	AssertFailed();
4080	break;
4081	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4082	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4083	{
4084	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4085	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4086	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4087	AssertRC(rc);
4088
4089	#ifdef VBOX_STRICT
4090	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4091	pVmxTransient->uExitReason));
4092	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4093	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4094	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4095	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4096	else
4097	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4098
4099	/* VMX control bits. */
4100	uint32_t u32Val;
4101	uint64_t u64Val;
4102	HMVMXHCUINTREG uHCReg;
4103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4104	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4106	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4108	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4110	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4112	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4114	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4116	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4118	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4120	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4121	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4122	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4123	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4124	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4125	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4126	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4128	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4129	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4130	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4131	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4132	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4133	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4134	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4135	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4136	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4137	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4138	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4139	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4140	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4141	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4142	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4143	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4144	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4145
4146	/* Guest bits. */
4147	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4148	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4149	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4150	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4151	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4152	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4153	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4154	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4155
4156	/* Host bits. */
4157	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4158	Log4(("Host CR0 %#RHr\n", uHCReg));
4159	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4160	Log4(("Host CR3 %#RHr\n", uHCReg));
4161	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4162	Log4(("Host CR4 %#RHr\n", uHCReg));
4163
4164	RTGDTR HostGdtr;
4165	PCX86DESCHC pDesc;
4166	ASMGetGDTR(&HostGdtr);
4167	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4168	Log4(("Host CS %#08x\n", u32Val));
4169	if (u32Val < HostGdtr.cbGdt)
4170	{
4171	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4172	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4173	}
4174
4175	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4176	Log4(("Host DS %#08x\n", u32Val));
4177	if (u32Val < HostGdtr.cbGdt)
4178	{
4179	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4180	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4181	}
4182
4183	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4184	Log4(("Host ES %#08x\n", u32Val));
4185	if (u32Val < HostGdtr.cbGdt)
4186	{
4187	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4188	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4189	}
4190
4191	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4192	Log4(("Host FS %#08x\n", u32Val));
4193	if (u32Val < HostGdtr.cbGdt)
4194	{
4195	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4196	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4197	}
4198
4199	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4200	Log4(("Host GS %#08x\n", u32Val));
4201	if (u32Val < HostGdtr.cbGdt)
4202	{
4203	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4204	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4205	}
4206
4207	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4208	Log4(("Host SS %#08x\n", u32Val));
4209	if (u32Val < HostGdtr.cbGdt)
4210	{
4211	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4212	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4213	}
4214
4215	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4216	Log4(("Host TR %#08x\n", u32Val));
4217	if (u32Val < HostGdtr.cbGdt)
4218	{
4219	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4220	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4221	}
4222
4223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4224	Log4(("Host TR Base %#RHv\n", uHCReg));
4225	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4226	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4227	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4228	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4229	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4230	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4231	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4232	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4233	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4234	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4235	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4236	Log4(("Host RSP %#RHv\n", uHCReg));
4237	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4238	Log4(("Host RIP %#RHv\n", uHCReg));
4239	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4240	if (HMVMX_IS_64BIT_HOST_MODE())
4241	{
4242	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4243	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4244	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4245	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4246	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4247	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4248	}
4249	# endif
4250	#endif /* VBOX_STRICT */
4251	break;
4252	}
4253
4254	default:
4255	/* Impossible */
4256	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4257	break;
4258	}
4259	NOREF(pVM);
4260	}
4261
4262
4263	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4264	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4265	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4266	#endif
4267	#ifdef VBOX_STRICT
4268	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4269	{
4270	switch (idxField)
4271	{
4272	case VMX_VMCS_GUEST_RIP:
4273	case VMX_VMCS_GUEST_RSP:
4274	case VMX_VMCS_GUEST_SYSENTER_EIP:
4275	case VMX_VMCS_GUEST_SYSENTER_ESP:
4276	case VMX_VMCS_GUEST_GDTR_BASE:
4277	case VMX_VMCS_GUEST_IDTR_BASE:
4278	case VMX_VMCS_GUEST_CS_BASE:
4279	case VMX_VMCS_GUEST_DS_BASE:
4280	case VMX_VMCS_GUEST_ES_BASE:
4281	case VMX_VMCS_GUEST_FS_BASE:
4282	case VMX_VMCS_GUEST_GS_BASE:
4283	case VMX_VMCS_GUEST_SS_BASE:
4284	case VMX_VMCS_GUEST_LDTR_BASE:
4285	case VMX_VMCS_GUEST_TR_BASE:
4286	case VMX_VMCS_GUEST_CR3:
4287	return true;
4288	}
4289	return false;
4290	}
4291
4292	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4293	{
4294	switch (idxField)
4295	{
4296	/* Read-only fields. */
4297	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4298	return true;
4299	}
4300	/* Remaining readable fields should also be writable. */
4301	return hmR0VmxIsValidWriteField(idxField);
4302	}
4303	#endif /* VBOX_STRICT */
4304
4305
4306	/**
4307	* Executes the specified handler in 64-bit mode.
4308	*
4309	* @returns VBox status code.
4310	* @param pVM Pointer to the VM.
4311	* @param pVCpu Pointer to the VMCPU.
4312	* @param pCtx Pointer to the guest CPU context.
4313	* @param enmOp The operation to perform.
4314	* @param cbParam Number of parameters.
4315	* @param paParam Array of 32-bit parameters.
4316	*/
4317	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4318	uint32_t *paParam)
4319	{
4320	int rc, rc2;
4321	PHMGLOBLCPUINFO pCpu;
4322	RTHCPHYS HCPhysCpuPage;
4323	RTCCUINTREG uOldEFlags;
4324
4325	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4326	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4327	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4328	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4329
4330	#ifdef VBOX_STRICT
4331	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4332	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4333
4334	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4335	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4336	#endif
4337
4338	/* Disable interrupts. */
4339	uOldEFlags = ASMIntDisableFlags();
4340
4341	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4342	RTCPUID idHostCpu = RTMpCpuId();
4343	CPUMR0SetLApic(pVM, idHostCpu);
4344	#endif
4345
4346	pCpu = HMR0GetCurrentCpu();
4347	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4348
4349	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4350	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4351
4352	/* Leave VMX Root Mode. */
4353	VMXDisable();
4354
4355	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4356
4357	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4358	CPUMSetHyperEIP(pVCpu, enmOp);
4359	for (int i = (int)cbParam - 1; i >= 0; i--)
4360	CPUMPushHyper(pVCpu, paParam[i]);
4361
4362	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4363
4364	/* Call the switcher. */
4365	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4366	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4367
4368	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4369	/* Make sure the VMX instructions don't cause #UD faults. */
4370	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4371
4372	/* Re-enter VMX Root Mode */
4373	rc2 = VMXEnable(HCPhysCpuPage);
4374	if (RT_FAILURE(rc2))
4375	{
4376	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4377	ASMSetFlags(uOldEFlags);
4378	return rc2;
4379	}
4380
4381	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4382	AssertRC(rc2);
4383	Assert(!(ASMGetFlags() & X86_EFL_IF));
4384	ASMSetFlags(uOldEFlags);
4385	return rc;
4386	}
4387
4388
4389	/**
4390	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4391	* supporting 64-bit guests.
4392	*
4393	* @returns VBox status code.
4394	* @param fResume Whether to VMLAUNCH or VMRESUME.
4395	* @param pCtx Pointer to the guest-CPU context.
4396	* @param pCache Pointer to the VMCS cache.
4397	* @param pVM Pointer to the VM.
4398	* @param pVCpu Pointer to the VMCPU.
4399	*/
4400	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4401	{
4402	uint32_t aParam[6];
4403	PHMGLOBLCPUINFO pCpu = NULL;
4404	RTHCPHYS HCPhysCpuPage = 0;
4405	int rc = VERR_INTERNAL_ERROR_5;
4406
4407	pCpu = HMR0GetCurrentCpu();
4408	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4409
4410	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4411	pCache->uPos = 1;
4412	pCache->interPD = PGMGetInterPaeCR3(pVM);
4413	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4414	#endif
4415
4416	#ifdef VBOX_STRICT
4417	pCache->TestIn.HCPhysCpuPage = 0;
4418	pCache->TestIn.HCPhysVmcs = 0;
4419	pCache->TestIn.pCache = 0;
4420	pCache->TestOut.HCPhysVmcs = 0;
4421	pCache->TestOut.pCache = 0;
4422	pCache->TestOut.pCtx = 0;
4423	pCache->TestOut.eflags = 0;
4424	#endif
4425
4426	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4427	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4428	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4429	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4430	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4431	aParam[5] = 0;
4432
4433	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4434	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4435	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4436	#endif
4437	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4438
4439	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4440	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4441	Assert(pCtx->dr[4] == 10);
4442	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4443	#endif
4444
4445	#ifdef VBOX_STRICT
4446	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4447	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4448	pVCpu->hm.s.vmx.HCPhysVmcs));
4449	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4450	pCache->TestOut.HCPhysVmcs));
4451	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4452	pCache->TestOut.pCache));
4453	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4454	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4455	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4456	pCache->TestOut.pCtx));
4457	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4458	#endif
4459	return rc;
4460	}
4461
4462
4463	/**
4464	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4465	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4466	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4467	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4468	*
4469	* @returns VBox status code.
4470	* @param pVM Pointer to the VM.
4471	* @param pVCpu Pointer to the VMCPU.
4472	*/
4473	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4474	{
4475	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4476	{ \
4477	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4478	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4479	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4480	++cReadFields; \
4481	}
4482
4483	AssertPtr(pVM);
4484	AssertPtr(pVCpu);
4485	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4486	uint32_t cReadFields = 0;
4487
4488	/*
4489	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4490	* and serve to indicate exceptions to the rules.
4491	*/
4492
4493	/* Guest-natural selector base fields. */
4494	#if 0
4495	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4498	#endif
4499	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4500	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4501	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4503	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4511	#if 0
4512	/* Unused natural width guest-state fields. */
4513	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4515	#endif
4516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4518
4519	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4520	#if 0
4521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4523	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4530	#endif
4531
4532	/* Natural width guest-state fields. */
4533	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4534	#if 0
4535	/* Currently unused field. */
4536	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4537	#endif
4538
4539	if (pVM->hm.s.fNestedPaging)
4540	{
4541	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4542	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4543	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4544	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4545	}
4546	else
4547	{
4548	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4549	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4550	}
4551
4552	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4553	return VINF_SUCCESS;
4554	}
4555
4556
4557	/**
4558	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4559	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4560	* darwin, running 64-bit guests).
4561	*
4562	* @returns VBox status code.
4563	* @param pVCpu Pointer to the VMCPU.
4564	* @param idxField The VMCS field encoding.
4565	* @param u64Val 16, 32 or 64 bits value.
4566	*/
4567	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4568	{
4569	int rc;
4570	switch (idxField)
4571	{
4572	/*
4573	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4574	*/
4575	/* 64-bit Control fields. */
4576	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4577	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4578	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4579	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4580	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4581	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4582	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4583	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4584	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4585	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4586	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4587	case VMX_VMCS64_CTRL_EPTP_FULL:
4588	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4589	/* 64-bit Guest-state fields. */
4590	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4591	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4592	case VMX_VMCS64_GUEST_PAT_FULL:
4593	case VMX_VMCS64_GUEST_EFER_FULL:
4594	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4595	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4596	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4597	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4598	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4599	/* 64-bit Host-state fields. */
4600	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4601	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4602	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4603	{
4604	rc = VMXWriteVmcs32(idxField, u64Val);
4605	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4606	break;
4607	}
4608
4609	/*
4610	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4611	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4612	*/
4613	/* Natural-width Guest-state fields. */
4614	case VMX_VMCS_GUEST_CR3:
4615	case VMX_VMCS_GUEST_ES_BASE:
4616	case VMX_VMCS_GUEST_CS_BASE:
4617	case VMX_VMCS_GUEST_SS_BASE:
4618	case VMX_VMCS_GUEST_DS_BASE:
4619	case VMX_VMCS_GUEST_FS_BASE:
4620	case VMX_VMCS_GUEST_GS_BASE:
4621	case VMX_VMCS_GUEST_LDTR_BASE:
4622	case VMX_VMCS_GUEST_TR_BASE:
4623	case VMX_VMCS_GUEST_GDTR_BASE:
4624	case VMX_VMCS_GUEST_IDTR_BASE:
4625	case VMX_VMCS_GUEST_RSP:
4626	case VMX_VMCS_GUEST_RIP:
4627	case VMX_VMCS_GUEST_SYSENTER_ESP:
4628	case VMX_VMCS_GUEST_SYSENTER_EIP:
4629	{
4630	if (!(u64Val >> 32))
4631	{
4632	/* If this field is 64-bit, VT-x will zero out the top bits. */
4633	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4634	}
4635	else
4636	{
4637	/* Assert that only the 32->64 switcher case should ever come here. */
4638	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4639	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4640	}
4641	break;
4642	}
4643
4644	default:
4645	{
4646	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4647	rc = VERR_INVALID_PARAMETER;
4648	break;
4649	}
4650	}
4651	AssertRCReturn(rc, rc);
4652	return rc;
4653	}
4654
4655
4656	/**
4657	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4658	* hosts (except darwin) for 64-bit guests.
4659	*
4660	* @param pVCpu Pointer to the VMCPU.
4661	* @param idxField The VMCS field encoding.
4662	* @param u64Val 16, 32 or 64 bits value.
4663	*/
4664	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4665	{
4666	AssertPtr(pVCpu);
4667	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4668
4669	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4670	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4671
4672	/* Make sure there are no duplicates. */
4673	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4674	{
4675	if (pCache->Write.aField[i] == idxField)
4676	{
4677	pCache->Write.aFieldVal[i] = u64Val;
4678	return VINF_SUCCESS;
4679	}
4680	}
4681
4682	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4683	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4684	pCache->Write.cValidEntries++;
4685	return VINF_SUCCESS;
4686	}
4687
4688	/* Enable later when the assembly code uses these as callbacks. */
4689	#if 0
4690	/*
4691	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4692	*
4693	* @param pVCpu Pointer to the VMCPU.
4694	* @param pCache Pointer to the VMCS cache.
4695	*
4696	* @remarks No-long-jump zone!!!
4697	*/
4698	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4699	{
4700	AssertPtr(pCache);
4701	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4702	{
4703	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4704	AssertRC(rc);
4705	}
4706	pCache->Write.cValidEntries = 0;
4707	}
4708
4709
4710	/**
4711	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4712	*
4713	* @param pVCpu Pointer to the VMCPU.
4714	* @param pCache Pointer to the VMCS cache.
4715	*
4716	* @remarks No-long-jump zone!!!
4717	*/
4718	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4719	{
4720	AssertPtr(pCache);
4721	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4722	{
4723	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4724	AssertRC(rc);
4725	}
4726	}
4727	#endif
4728	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4729
4730
4731	/**
4732	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4733	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4734	* timer.
4735	*
4736	* @returns VBox status code.
4737	* @param pVCpu Pointer to the VMCPU.
4738	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4739	* out-of-sync. Make sure to update the required fields
4740	* before using them.
4741	* @remarks No-long-jump zone!!!
4742	*/
4743	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4744	{
4745	int rc = VERR_INTERNAL_ERROR_5;
4746	bool fOffsettedTsc = false;
4747	PVM pVM = pVCpu->CTX_SUFF(pVM);
4748	if (pVM->hm.s.vmx.fUsePreemptTimer)
4749	{
4750	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4751
4752	/* Make sure the returned values have sane upper and lower boundaries. */
4753	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4754	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4755	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4756	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4757
4758	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4759	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4760	}
4761	else
4762	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4763
4764	if (fOffsettedTsc)
4765	{
4766	uint64_t u64CurTSC = ASMReadTSC();
4767	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4768	{
4769	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4770	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4771
4772	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4773	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4774	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4775	}
4776	else
4777	{
4778	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4779	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4780	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4781	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4782	}
4783	}
4784	else
4785	{
4786	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4787	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4788	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4789	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4790	}
4791	}
4792
4793
4794	/**
4795	* Determines if an exception is a contributory exception. Contributory
4796	* exceptions are ones which can cause double-faults. Page-fault is
4797	* intentionally not included here as it's a conditional contributory exception.
4798	*
4799	* @returns true if the exception is contributory, false otherwise.
4800	* @param uVector The exception vector.
4801	*/
4802	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4803	{
4804	switch (uVector)
4805	{
4806	case X86_XCPT_GP:
4807	case X86_XCPT_SS:
4808	case X86_XCPT_NP:
4809	case X86_XCPT_TS:
4810	case X86_XCPT_DE:
4811	return true;
4812	default:
4813	break;
4814	}
4815	return false;
4816	}
4817
4818
4819	/**
4820	* Sets an event as a pending event to be injected into the guest.
4821	*
4822	* @param pVCpu Pointer to the VMCPU.
4823	* @param u32IntrInfo The VM-entry interruption-information field.
4824	* @param cbInstr The VM-entry instruction length in bytes (for software
4825	* interrupts, exceptions and privileged software
4826	* exceptions).
4827	* @param u32ErrCode The VM-entry exception error code.
4828	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4829	* page-fault.
4830	*
4831	* @remarks Statistics counter assumes this is a guest event being injected or
4832	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4833	* always incremented.
4834	*/
4835	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4836	RTGCUINTPTR GCPtrFaultAddress)
4837	{
4838	Assert(!pVCpu->hm.s.Event.fPending);
4839	pVCpu->hm.s.Event.fPending = true;
4840	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4841	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4842	pVCpu->hm.s.Event.cbInstr = cbInstr;
4843	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4844
4845	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4846	}
4847
4848
4849	/**
4850	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4851	*
4852	* @param pVCpu Pointer to the VMCPU.
4853	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4854	* out-of-sync. Make sure to update the required fields
4855	* before using them.
4856	*/
4857	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4858	{
4859	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4860	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4861	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4862	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4863	}
4864
4865
4866	/**
4867	* Handle a condition that occurred while delivering an event through the guest
4868	* IDT.
4869	*
4870	* @returns VBox status code (informational error codes included).
4871	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4872	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4873	* continue execution of the guest which will delivery the #DF.
4874	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4875	*
4876	* @param pVCpu Pointer to the VMCPU.
4877	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4878	* out-of-sync. Make sure to update the required fields
4879	* before using them.
4880	* @param pVmxTransient Pointer to the VMX transient structure.
4881	*
4882	* @remarks No-long-jump zone!!!
4883	*/
4884	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4885	{
4886	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4887	AssertRC(rc);
4888	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4889	{
4890	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4891	AssertRCReturn(rc, rc);
4892
4893	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4894	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4895	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4896
4897	typedef enum
4898	{
4899	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4900	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4901	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4902	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4903	} VMXREFLECTXCPT;
4904
4905	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4906	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4907	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4908	{
4909	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4910	{
4911	enmReflect = VMXREFLECTXCPT_XCPT;
4912	#ifdef VBOX_STRICT
4913	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4914	&& uExitVector == X86_XCPT_PF)
4915	{
4916	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4917	}
4918	#endif
4919	if ( uExitVector == X86_XCPT_PF
4920	&& uIdtVector == X86_XCPT_PF)
4921	{
4922	pVmxTransient->fVectoringPF = true;
4923	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4924	}
4925	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4926	&& hmR0VmxIsContributoryXcpt(uExitVector)
4927	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4928	\|\| uIdtVector == X86_XCPT_PF))
4929	{
4930	enmReflect = VMXREFLECTXCPT_DF;
4931	}
4932	else if (uIdtVector == X86_XCPT_DF)
4933	enmReflect = VMXREFLECTXCPT_TF;
4934	}
4935	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4936	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4937	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4938	{
4939	/*
4940	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4941	* (whatever they are) as they reoccur when restarting the instruction.
4942	*/
4943	enmReflect = VMXREFLECTXCPT_XCPT;
4944	}
4945	}
4946	else
4947	{
4948	/*
4949	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4950	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4951	* original exception to the guest after handling the VM-exit.
4952	*/
4953	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4954	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4955	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4956	{
4957	enmReflect = VMXREFLECTXCPT_XCPT;
4958	}
4959	}
4960
4961	switch (enmReflect)
4962	{
4963	case VMXREFLECTXCPT_XCPT:
4964	{
4965	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4966	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4967	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
4968
4969	uint32_t u32ErrCode = 0;
4970	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
4971	{
4972	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4973	AssertRCReturn(rc, rc);
4974	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4975	}
4976
4977	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4978	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4979	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4980	rc = VINF_SUCCESS;
4981	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
4982	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4983
4984	break;
4985	}
4986
4987	case VMXREFLECTXCPT_DF:
4988	{
4989	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4990	rc = VINF_HM_DOUBLE_FAULT;
4991	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
4992	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4993
4994	break;
4995	}
4996
4997	case VMXREFLECTXCPT_TF:
4998	{
4999	rc = VINF_EM_RESET;
5000	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5001	uExitVector));
5002	break;
5003	}
5004
5005	default:
5006	Assert(rc == VINF_SUCCESS);
5007	break;
5008	}
5009	}
5010	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5011	return rc;
5012	}
5013
5014
5015	/**
5016	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5017	*
5018	* @returns VBox status code.
5019	* @param pVCpu Pointer to the VMCPU.
5020	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5021	* out-of-sync. Make sure to update the required fields
5022	* before using them.
5023	*
5024	* @remarks No-long-jump zone!!!
5025	*/
5026	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5027	{
5028	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5029	{
5030	uint32_t uVal = 0;
5031	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5032	AssertRCReturn(rc, rc);
5033	uint32_t uShadow = 0;
5034	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5035	AssertRCReturn(rc, rc);
5036
5037	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5038	CPUMSetGuestCR0(pVCpu, uVal);
5039	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5040	}
5041	return VINF_SUCCESS;
5042	}
5043
5044
5045	/**
5046	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5047	*
5048	* @returns VBox status code.
5049	* @param pVCpu Pointer to the VMCPU.
5050	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5051	* out-of-sync. Make sure to update the required fields
5052	* before using them.
5053	*
5054	* @remarks No-long-jump zone!!!
5055	*/
5056	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5057	{
5058	int rc = VINF_SUCCESS;
5059	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5060	{
5061	uint32_t uVal = 0;
5062	uint32_t uShadow = 0;
5063	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5064	AssertRCReturn(rc, rc);
5065	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5066	AssertRCReturn(rc, rc);
5067
5068	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5069	CPUMSetGuestCR4(pVCpu, uVal);
5070	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5071	}
5072	return rc;
5073	}
5074
5075
5076	/**
5077	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5078	*
5079	* @returns VBox status code.
5080	* @param pVCpu Pointer to the VMCPU.
5081	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5082	* out-of-sync. Make sure to update the required fields
5083	* before using them.
5084	*
5085	* @remarks No-long-jump zone!!!
5086	*/
5087	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5088	{
5089	int rc = VINF_SUCCESS;
5090	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5091	{
5092	uint64_t u64Val = 0;
5093	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5094	AssertRCReturn(rc, rc);
5095
5096	pMixedCtx->rip = u64Val;
5097	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5098	}
5099	return rc;
5100	}
5101
5102
5103	/**
5104	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5105	*
5106	* @returns VBox status code.
5107	* @param pVCpu Pointer to the VMCPU.
5108	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5109	* out-of-sync. Make sure to update the required fields
5110	* before using them.
5111	*
5112	* @remarks No-long-jump zone!!!
5113	*/
5114	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5115	{
5116	int rc = VINF_SUCCESS;
5117	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5118	{
5119	uint64_t u64Val = 0;
5120	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5121	AssertRCReturn(rc, rc);
5122
5123	pMixedCtx->rsp = u64Val;
5124	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5125	}
5126	return rc;
5127	}
5128
5129
5130	/**
5131	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5132	*
5133	* @returns VBox status code.
5134	* @param pVCpu Pointer to the VMCPU.
5135	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5136	* out-of-sync. Make sure to update the required fields
5137	* before using them.
5138	*
5139	* @remarks No-long-jump zone!!!
5140	*/
5141	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5142	{
5143	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5144	{
5145	uint32_t uVal = 0;
5146	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5147	AssertRCReturn(rc, rc);
5148
5149	pMixedCtx->eflags.u32 = uVal;
5150	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5151	{
5152	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5153	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5154
5155	pMixedCtx->eflags.Bits.u1VM = 0;
5156	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5157	}
5158
5159	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5160	}
5161	return VINF_SUCCESS;
5162	}
5163
5164
5165	/**
5166	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5167	* guest-CPU context.
5168	*/
5169	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5170	{
5171	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5172	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5173	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5174	return rc;
5175	}
5176
5177
5178	/**
5179	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5180	* from the guest-state area in the VMCS.
5181	*
5182	* @param pVCpu Pointer to the VMCPU.
5183	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5184	* out-of-sync. Make sure to update the required fields
5185	* before using them.
5186	*
5187	* @remarks No-long-jump zone!!!
5188	*/
5189	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5190	{
5191	uint32_t uIntrState = 0;
5192	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5193	AssertRC(rc);
5194
5195	if (!uIntrState)
5196	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5197	else
5198	{
5199	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5200	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5201	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5202	AssertRC(rc);
5203	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5204	AssertRC(rc);
5205
5206	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5207	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5208	}
5209	}
5210
5211
5212	/**
5213	* Saves the guest's activity state.
5214	*
5215	* @returns VBox status code.
5216	* @param pVCpu Pointer to the VMCPU.
5217	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5218	* out-of-sync. Make sure to update the required fields
5219	* before using them.
5220	*
5221	* @remarks No-long-jump zone!!!
5222	*/
5223	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5224	{
5225	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5226	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5227	return VINF_SUCCESS;
5228	}
5229
5230
5231	/**
5232	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5233	* the current VMCS into the guest-CPU context.
5234	*
5235	* @returns VBox status code.
5236	* @param pVCpu Pointer to the VMCPU.
5237	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5238	* out-of-sync. Make sure to update the required fields
5239	* before using them.
5240	*
5241	* @remarks No-long-jump zone!!!
5242	*/
5243	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5244	{
5245	int rc = VINF_SUCCESS;
5246	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5247	{
5248	uint32_t u32Val = 0;
5249	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5250	pMixedCtx->SysEnter.cs = u32Val;
5251	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5252	}
5253
5254	uint64_t u64Val = 0;
5255	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5256	{
5257	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5258	pMixedCtx->SysEnter.eip = u64Val;
5259	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5260	}
5261	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5262	{
5263	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5264	pMixedCtx->SysEnter.esp = u64Val;
5265	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5266	}
5267	return rc;
5268	}
5269
5270
5271	/**
5272	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5273	* context.
5274	*
5275	* @returns VBox status code.
5276	* @param pVCpu Pointer to the VMCPU.
5277	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5278	* out-of-sync. Make sure to update the required fields
5279	* before using them.
5280	*
5281	* @remarks No-long-jump zone!!!
5282	*/
5283	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5284	{
5285	int rc = VINF_SUCCESS;
5286	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5287	{
5288	uint64_t u64Val = 0;
5289	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5290	pMixedCtx->fs.u64Base = u64Val;
5291	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5292	}
5293	return rc;
5294	}
5295
5296
5297	/**
5298	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5299	* context.
5300	*
5301	* @returns VBox status code.
5302	* @param pVCpu Pointer to the VMCPU.
5303	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5304	* out-of-sync. Make sure to update the required fields
5305	* before using them.
5306	*
5307	* @remarks No-long-jump zone!!!
5308	*/
5309	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5310	{
5311	int rc = VINF_SUCCESS;
5312	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5313	{
5314	uint64_t u64Val = 0;
5315	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5316	pMixedCtx->gs.u64Base = u64Val;
5317	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5318	}
5319	return rc;
5320	}
5321
5322
5323	/**
5324	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5325	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5326	* and TSC_AUX.
5327	*
5328	* @returns VBox status code.
5329	* @param pVCpu Pointer to the VMCPU.
5330	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5331	* out-of-sync. Make sure to update the required fields
5332	* before using them.
5333	*
5334	* @remarks No-long-jump zone!!!
5335	*/
5336	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5337	{
5338	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5339	return VINF_SUCCESS;
5340
5341	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5342	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5343	{
5344	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5345	pMsr += i;
5346	switch (pMsr->u32IndexMSR)
5347	{
5348	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5349	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5350	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5351	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5352	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5353	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5354	default:
5355	{
5356	AssertFailed();
5357	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5358	}
5359	}
5360	}
5361	#endif
5362
5363	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5364	return VINF_SUCCESS;
5365	}
5366
5367
5368	/**
5369	* Saves the guest control registers from the current VMCS into the guest-CPU
5370	* context.
5371	*
5372	* @returns VBox status code.
5373	* @param pVCpu Pointer to the VMCPU.
5374	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5375	* out-of-sync. Make sure to update the required fields
5376	* before using them.
5377	*
5378	* @remarks No-long-jump zone!!!
5379	*/
5380	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5381	{
5382	/* Guest CR0. Guest FPU. */
5383	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5384	AssertRCReturn(rc, rc);
5385
5386	/* Guest CR4. */
5387	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5388	AssertRCReturn(rc, rc);
5389
5390	/* Guest CR2 - updated always during the world-switch or in #PF. */
5391	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5392	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5393	{
5394	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5395	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5396
5397	PVM pVM = pVCpu->CTX_SUFF(pVM);
5398	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5399	\|\| ( pVM->hm.s.fNestedPaging
5400	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5401	{
5402	uint64_t u64Val = 0;
5403	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5404	if (pMixedCtx->cr3 != u64Val)
5405	{
5406	CPUMSetGuestCR3(pVCpu, u64Val);
5407	if (VMMRZCallRing3IsEnabled(pVCpu))
5408	{
5409	PGMUpdateCR3(pVCpu, u64Val);
5410	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5411	}
5412	else
5413	{
5414	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5415	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5416	}
5417	}
5418
5419	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5420	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5421	{
5422	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5423	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5424	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5425	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5426
5427	if (VMMRZCallRing3IsEnabled(pVCpu))
5428	{
5429	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5430	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5431	}
5432	else
5433	{
5434	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5435	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5436	}
5437	}
5438	}
5439
5440	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5441	}
5442
5443	/*
5444	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5445	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5446	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5447	*
5448	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5449	*/
5450	if (VMMRZCallRing3IsEnabled(pVCpu))
5451	{
5452	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5453	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5454
5455	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5456	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5457
5458	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5459	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5460	}
5461
5462	return rc;
5463	}
5464
5465
5466	/**
5467	* Reads a guest segment register from the current VMCS into the guest-CPU
5468	* context.
5469	*
5470	* @returns VBox status code.
5471	* @param pVCpu Pointer to the VMCPU.
5472	* @param idxSel Index of the selector in the VMCS.
5473	* @param idxLimit Index of the segment limit in the VMCS.
5474	* @param idxBase Index of the segment base in the VMCS.
5475	* @param idxAccess Index of the access rights of the segment in the VMCS.
5476	* @param pSelReg Pointer to the segment selector.
5477	*
5478	* @remarks No-long-jump zone!!!
5479	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5480	* macro as that takes care of whether to read from the VMCS cache or
5481	* not.
5482	*/
5483	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5484	PCPUMSELREG pSelReg)
5485	{
5486	uint32_t u32Val = 0;
5487	int rc = VMXReadVmcs32(idxSel, &u32Val);
5488	AssertRCReturn(rc, rc);
5489	pSelReg->Sel = (uint16_t)u32Val;
5490	pSelReg->ValidSel = (uint16_t)u32Val;
5491	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5492
5493	rc = VMXReadVmcs32(idxLimit, &u32Val);
5494	AssertRCReturn(rc, rc);
5495	pSelReg->u32Limit = u32Val;
5496
5497	uint64_t u64Val = 0;
5498	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5499	AssertRCReturn(rc, rc);
5500	pSelReg->u64Base = u64Val;
5501
5502	rc = VMXReadVmcs32(idxAccess, &u32Val);
5503	AssertRCReturn(rc, rc);
5504	pSelReg->Attr.u = u32Val;
5505
5506	/*
5507	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5508	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5509	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5510	*
5511	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5512	*
5513	* bird: This isn't quite as simple. VT-x and VBox(!) requires the DPL for SS to be the same as CPL. In 64-bit mode it
5514	* is possible (int/trap/xxx injects does this when switching rings) to load SS with a NULL selector and RPL=CPL.
5515	* The Attr.u = X86DESCATTR_UNUSABLE works fine as long as nobody uses ring-1 or ring-2. VT-x updates the DPL
5516	* correctly in the attributes of SS even when the unusable bit is set, we need to preserve the DPL or we get invalid
5517	* guest state trouble. Try bs2-cpu-hidden-regs-1.
5518	*/
5519	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5520	{
5521	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5522	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x\n", idxSel, pSelReg->Attr.u));
5523
5524	if (idxSel == VMX_VMCS16_GUEST_FIELD_SS)
5525	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_DPL;
5526	else if (idxSel == VMX_VMCS16_GUEST_FIELD_CS)
5527	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G;
5528	else
5529	pSelReg->Attr.u = X86DESCATTR_UNUSABLE;
5530	}
5531	return VINF_SUCCESS;
5532	}
5533
5534
5535	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5536	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5537	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5538	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5539	#else
5540	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5541	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5542	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5543	#endif
5544
5545
5546	/**
5547	* Saves the guest segment registers from the current VMCS into the guest-CPU
5548	* context.
5549	*
5550	* @returns VBox status code.
5551	* @param pVCpu Pointer to the VMCPU.
5552	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5553	* out-of-sync. Make sure to update the required fields
5554	* before using them.
5555	*
5556	* @remarks No-long-jump zone!!!
5557	*/
5558	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5559	{
5560	/* Guest segment registers. */
5561	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5562	{
5563	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5564	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5565	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5566	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5567	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5568	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5569	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5570
5571	/* Restore segment attributes for real-on-v86 mode hack. */
5572	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5573	{
5574	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5575	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5576	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5577	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5578	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5579	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5580	}
5581	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5582	}
5583
5584	return VINF_SUCCESS;
5585	}
5586
5587
5588	/**
5589	* Saves the guest descriptor table registers and task register from the current
5590	* VMCS into the guest-CPU context.
5591	*
5592	* @returns VBox status code.
5593	* @param pVCpu Pointer to the VMCPU.
5594	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5595	* out-of-sync. Make sure to update the required fields
5596	* before using them.
5597	*
5598	* @remarks No-long-jump zone!!!
5599	*/
5600	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5601	{
5602	int rc = VINF_SUCCESS;
5603
5604	/* Guest LDTR. */
5605	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5606	{
5607	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5608	AssertRCReturn(rc, rc);
5609	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5610	}
5611
5612	/* Guest GDTR. */
5613	uint64_t u64Val = 0;
5614	uint32_t u32Val = 0;
5615	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5616	{
5617	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5618	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5619	pMixedCtx->gdtr.pGdt = u64Val;
5620	pMixedCtx->gdtr.cbGdt = u32Val;
5621	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5622	}
5623
5624	/* Guest IDTR. */
5625	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5626	{
5627	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5628	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5629	pMixedCtx->idtr.pIdt = u64Val;
5630	pMixedCtx->idtr.cbIdt = u32Val;
5631	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5632	}
5633
5634	/* Guest TR. */
5635	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5636	{
5637	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5638	AssertRCReturn(rc, rc);
5639
5640	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5641	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5642	{
5643	rc = VMXLOCAL_READ_SEG(TR, tr);
5644	AssertRCReturn(rc, rc);
5645	}
5646	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5647	}
5648	return rc;
5649	}
5650
5651	#undef VMXLOCAL_READ_SEG
5652
5653
5654	/**
5655	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5656	* context.
5657	*
5658	* @returns VBox status code.
5659	* @param pVCpu Pointer to the VMCPU.
5660	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5661	* out-of-sync. Make sure to update the required fields
5662	* before using them.
5663	*
5664	* @remarks No-long-jump zone!!!
5665	*/
5666	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5667	{
5668	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5669	{
5670	if (!CPUMIsHyperDebugStateActive(pVCpu))
5671	{
5672	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5673	uint32_t u32Val;
5674	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5675	pMixedCtx->dr[7] = u32Val;
5676	}
5677
5678	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5679	}
5680	return VINF_SUCCESS;
5681	}
5682
5683
5684	/**
5685	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5686	*
5687	* @returns VBox status code.
5688	* @param pVCpu Pointer to the VMCPU.
5689	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5690	* out-of-sync. Make sure to update the required fields
5691	* before using them.
5692	*
5693	* @remarks No-long-jump zone!!!
5694	*/
5695	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5696	{
5697	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5698	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5699	return VINF_SUCCESS;
5700	}
5701
5702
5703	/**
5704	* Saves the entire guest state from the currently active VMCS into the
5705	* guest-CPU context. This essentially VMREADs all guest-data.
5706	*
5707	* @returns VBox status code.
5708	* @param pVCpu Pointer to the VMCPU.
5709	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5710	* out-of-sync. Make sure to update the required fields
5711	* before using them.
5712	*/
5713	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5714	{
5715	Assert(pVCpu);
5716	Assert(pMixedCtx);
5717
5718	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5719	return VINF_SUCCESS;
5720
5721	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5722	there is no real need to. */
5723	if (VMMRZCallRing3IsEnabled(pVCpu))
5724	VMMR0LogFlushDisable(pVCpu);
5725	else
5726	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5727	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5728
5729	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5730	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5731
5732	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5733	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5734
5735	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5736	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5737
5738	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5739	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5740
5741	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5742	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5743
5744	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5745	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5746
5747	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5748	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5749
5750	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5751	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5752
5753	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5754	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5755
5756	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5757	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5758
5759	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5760	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5761
5762	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5763	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5764
5765	if (VMMRZCallRing3IsEnabled(pVCpu))
5766	VMMR0LogFlushEnable(pVCpu);
5767
5768	return rc;
5769	}
5770
5771
5772	/**
5773	* Check per-VM and per-VCPU force flag actions that require us to go back to
5774	* ring-3 for one reason or another.
5775	*
5776	* @returns VBox status code (information status code included).
5777	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5778	* ring-3.
5779	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5780	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5781	* interrupts)
5782	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5783	* all EMTs to be in ring-3.
5784	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5785	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5786	* to the EM loop.
5787	*
5788	* @param pVM Pointer to the VM.
5789	* @param pVCpu Pointer to the VMCPU.
5790	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5791	* out-of-sync. Make sure to update the required fields
5792	* before using them.
5793	*/
5794	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5795	{
5796	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5797
5798	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5799	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5800	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5801	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5802	{
5803	/* We need the control registers now, make sure the guest-CPU context is updated. */
5804	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5805	AssertRCReturn(rc3, rc3);
5806
5807	/* Pending HM CR3 sync. */
5808	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5809	{
5810	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5811	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
5812	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5813	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5814	}
5815
5816	/* Pending HM PAE PDPEs. */
5817	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5818	{
5819	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5820	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5821	}
5822
5823	/* Pending PGM C3 sync. */
5824	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5825	{
5826	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5827	if (rc2 != VINF_SUCCESS)
5828	{
5829	AssertRC(rc2);
5830	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5831	return rc2;
5832	}
5833	}
5834
5835	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5836	/* -XXX- what was that about single stepping? */
5837	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5838	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5839	{
5840	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5841	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5842	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5843	return rc2;
5844	}
5845
5846	/* Pending VM request packets, such as hardware interrupts. */
5847	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5848	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5849	{
5850	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5851	return VINF_EM_PENDING_REQUEST;
5852	}
5853
5854	/* Pending PGM pool flushes. */
5855	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5856	{
5857	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5858	return VINF_PGM_POOL_FLUSH_PENDING;
5859	}
5860
5861	/* Pending DMA requests. */
5862	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5863	{
5864	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5865	return VINF_EM_RAW_TO_R3;
5866	}
5867	}
5868
5869	/* Paranoia. */
5870	return VINF_SUCCESS;
5871	}
5872
5873
5874	/**
5875	* Converts any TRPM trap into a pending HM event. This is typically used when
5876	* entering from ring-3 (not longjmp returns).
5877	*
5878	* @param pVCpu Pointer to the VMCPU.
5879	*/
5880	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5881	{
5882	Assert(TRPMHasTrap(pVCpu));
5883	Assert(!pVCpu->hm.s.Event.fPending);
5884
5885	uint8_t uVector;
5886	TRPMEVENT enmTrpmEvent;
5887	RTGCUINT uErrCode;
5888	RTGCUINTPTR GCPtrFaultAddress;
5889	uint8_t cbInstr;
5890
5891	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5892	AssertRC(rc);
5893
5894	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5895	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5896	if (enmTrpmEvent == TRPM_TRAP)
5897	{
5898	switch (uVector)
5899	{
5900	case X86_XCPT_BP:
5901	case X86_XCPT_OF:
5902	{
5903	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5904	break;
5905	}
5906
5907	case X86_XCPT_PF:
5908	case X86_XCPT_DF:
5909	case X86_XCPT_TS:
5910	case X86_XCPT_NP:
5911	case X86_XCPT_SS:
5912	case X86_XCPT_GP:
5913	case X86_XCPT_AC:
5914	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5915	/* no break! */
5916	default:
5917	{
5918	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5919	break;
5920	}
5921	}
5922	}
5923	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5924	{
5925	if (uVector == X86_XCPT_NMI)
5926	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5927	else
5928	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5929	}
5930	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5931	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5932	else
5933	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5934
5935	rc = TRPMResetTrap(pVCpu);
5936	AssertRC(rc);
5937	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5938	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5939
5940	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5941	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
5942	}
5943
5944
5945	/**
5946	* Converts any pending HM event into a TRPM trap. Typically used when leaving
5947	* VT-x to execute any instruction.
5948	*
5949	* @param pvCpu Pointer to the VMCPU.
5950	*/
5951	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5952	{
5953	Assert(pVCpu->hm.s.Event.fPending);
5954
5955	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5956	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5957	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5958	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5959
5960	/* If a trap was already pending, we did something wrong! */
5961	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5962
5963	TRPMEVENT enmTrapType;
5964	switch (uVectorType)
5965	{
5966	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5967	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5968	enmTrapType = TRPM_HARDWARE_INT;
5969	break;
5970	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5971	enmTrapType = TRPM_SOFTWARE_INT;
5972	break;
5973	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5974	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5975	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5976	enmTrapType = TRPM_TRAP;
5977	break;
5978	default:
5979	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5980	enmTrapType = TRPM_32BIT_HACK;
5981	break;
5982	}
5983
5984	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5985
5986	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5987	AssertRC(rc);
5988
5989	if (fErrorCodeValid)
5990	TRPMSetErrorCode(pVCpu, uErrorCode);
5991
5992	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5993	&& uVector == X86_XCPT_PF)
5994	{
5995	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5996	}
5997	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5998	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5999	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6000	{
6001	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6002	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6003	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6004	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6005	}
6006	pVCpu->hm.s.Event.fPending = false;
6007	}
6008
6009
6010	/**
6011	* Does the necessary state syncing before returning to ring-3 for any reason
6012	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6013	*
6014	* @param pVM Pointer to the VM.
6015	* @param pVCpu Pointer to the VMCPU.
6016	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6017	* out-of-sync. Make sure to update the required fields
6018	* before using them.
6019	*
6020	* @remarks No-long-jmp zone!!!
6021	*/
6022	static void hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6023	{
6024	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6025	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6026
6027	/* Save the guest state if necessary. */
6028	if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6029	{
6030	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6031	AssertRC(rc);
6032	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6033	}
6034
6035	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6036	if (CPUMIsGuestFPUStateActive(pVCpu))
6037	{
6038	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6039	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6040	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
6041	}
6042
6043	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6044	#ifdef VBOX_STRICT
6045	if (CPUMIsHyperDebugStateActive(pVCpu))
6046	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6047	#endif
6048	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6049	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
6050	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6051	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6052
6053	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6054	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6055	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6056	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6057	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6058	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6059	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6060	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6061
6062	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6063	}
6064
6065
6066	/**
6067	* Does the necessary state syncing before doing a longjmp to ring-3.
6068	*
6069	* @param pVM Pointer to the VM.
6070	* @param pVCpu Pointer to the VMCPU.
6071	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6072	* out-of-sync. Make sure to update the required fields
6073	* before using them.
6074	*
6075	* @remarks No-long-jmp zone!!!
6076	*/
6077	DECLINLINE(void) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6078	{
6079	hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6080	}
6081
6082
6083	/**
6084	* Take necessary actions before going back to ring-3.
6085	*
6086	* An action requires us to go back to ring-3. This function does the necessary
6087	* steps before we can safely return to ring-3. This is not the same as longjmps
6088	* to ring-3, this is voluntary and prepares the guest so it may continue
6089	* executing outside HM (recompiler/IEM).
6090	*
6091	* @param pVM Pointer to the VM.
6092	* @param pVCpu Pointer to the VMCPU.
6093	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6094	* out-of-sync. Make sure to update the required fields
6095	* before using them.
6096	* @param rcExit The reason for exiting to ring-3. Can be
6097	* VINF_VMM_UNKNOWN_RING3_CALL.
6098	*/
6099	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6100	{
6101	Assert(pVM);
6102	Assert(pVCpu);
6103	Assert(pMixedCtx);
6104	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6105
6106	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6107	{
6108	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
6109	return;
6110	}
6111	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6112	{
6113	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6114	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6115	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6116	pVCpu->hm.s.vmx.LastError.idCurrentCpu = RTMpCpuId();
6117	return;
6118	}
6119
6120	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6121	VMMRZCallRing3Disable(pVCpu);
6122	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6123
6124	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6125	if (pVCpu->hm.s.Event.fPending)
6126	{
6127	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6128	Assert(!pVCpu->hm.s.Event.fPending);
6129	}
6130
6131	/* Save guest state and restore host state bits. */
6132	hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6133	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6134
6135	/* Sync recompiler state. */
6136	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6137	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6138	\| CPUM_CHANGED_LDTR
6139	\| CPUM_CHANGED_GDTR
6140	\| CPUM_CHANGED_IDTR
6141	\| CPUM_CHANGED_TR
6142	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6143	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6144	if ( pVM->hm.s.fNestedPaging
6145	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6146	{
6147	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6148	}
6149
6150	/* On our way back from ring-3 the following needs to be done. */
6151	/** @todo This can change with preemption hooks. */
6152	if (rcExit == VINF_EM_RAW_INTERRUPT)
6153	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
6154	else
6155	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
6156
6157	/* Make sure we've undo the trap flag if we tried to single step something. */
6158	if (pVCpu->hm.s.fClearTrapFlag)
6159	{
6160	pVCpu->hm.s.fClearTrapFlag = false;
6161	pMixedCtx->eflags.Bits.u1TF = 0;
6162	}
6163
6164	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6165	VMMRZCallRing3Enable(pVCpu);
6166	}
6167
6168
6169	/**
6170	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6171	* longjump to ring-3 and possibly get preempted.
6172	*
6173	* @param pVCpu Pointer to the VMCPU.
6174	* @param enmOperation The operation causing the ring-3 longjump.
6175	* @param pvUser The user argument (pointer to the possibly
6176	* out-of-date guest-CPU context).
6177	*
6178	* @remarks Must never be called with @a enmOperation ==
6179	* VMMCALLRING3_VM_R0_ASSERTION.
6180	*/
6181	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6182	{
6183	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6184	Assert(pVCpu);
6185	Assert(pvUser);
6186	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6187	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6188
6189	VMMRZCallRing3Disable(pVCpu);
6190	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6191	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6192	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6193	VMMRZCallRing3Enable(pVCpu);
6194	}
6195
6196
6197	/**
6198	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6199	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6200	*
6201	* @param pVCpu Pointer to the VMCPU.
6202	*/
6203	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6204	{
6205	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6206	{
6207	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6208	{
6209	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6210	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6211	AssertRC(rc);
6212	}
6213	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6214	}
6215
6216
6217	/**
6218	* Injects any pending events into the guest if the guest is in a state to
6219	* receive them.
6220	*
6221	* @returns VBox status code (informational status codes included).
6222	* @param pVCpu Pointer to the VMCPU.
6223	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6224	* out-of-sync. Make sure to update the required fields
6225	* before using them.
6226	*/
6227	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6228	{
6229	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6230	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6231	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6232	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6233
6234	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6235	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6236	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6237	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6238	Assert(!TRPMHasTrap(pVCpu));
6239
6240	int rc = VINF_SUCCESS;
6241	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6242	{
6243	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6244	bool fInject = true;
6245	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6246	{
6247	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6248	AssertRCReturn(rc, rc);
6249	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6250	if ( fBlockInt
6251	\|\| fBlockSti
6252	\|\| fBlockMovSS)
6253	{
6254	fInject = false;
6255	}
6256	}
6257	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6258	&& ( fBlockMovSS
6259	\|\| fBlockSti))
6260	{
6261	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6262	fInject = false;
6263	}
6264
6265	if (fInject)
6266	{
6267	Log4(("Injecting pending event vcpu[%RU32]\n", pVCpu->idCpu));
6268	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6269	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6270	AssertRCReturn(rc, rc);
6271	pVCpu->hm.s.Event.fPending = false;
6272
6273	#ifdef VBOX_WITH_STATISTICS
6274	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6275	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6276	else
6277	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6278	#endif
6279	}
6280	else
6281	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6282	} /** @todo SMI. SMIs take priority over NMIs. */
6283	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6284	{
6285	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6286	if ( !fBlockMovSS
6287	&& !fBlockSti)
6288	{
6289	Log4(("Injecting NMI\n"));
6290	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6291	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6292	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6293	0 /* GCPtrFaultAddress */, &uIntrState);
6294	AssertRCReturn(rc, rc);
6295	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6296
6297	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6298	}
6299	else
6300	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6301	}
6302	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6303	{
6304	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6305	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6306	AssertRCReturn(rc, rc);
6307	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6308	if ( !fBlockInt
6309	&& !fBlockSti
6310	&& !fBlockMovSS)
6311	{
6312	uint8_t u8Interrupt;
6313	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6314	if (RT_SUCCESS(rc))
6315	{
6316	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6317	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6318	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6319	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6320	0 /* GCPtrFaultAddress */, &uIntrState);
6321
6322	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6323	}
6324	else
6325	{
6326	/** @todo Does this actually happen? If not turn it into an assertion. */
6327	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6328	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6329	rc = VINF_SUCCESS;
6330	}
6331	}
6332	else
6333	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6334	}
6335
6336	/*
6337	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6338	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6339	*/
6340	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6341	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6342	int rc2 = VINF_SUCCESS;
6343	if ( fBlockSti
6344	\|\| fBlockMovSS)
6345	{
6346	if (!DBGFIsStepping(pVCpu))
6347	{
6348	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6349	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6350	{
6351	/*
6352	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6353	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6354	* See Intel spec. 27.3.4 "Saving Non-Register State".
6355	*/
6356	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6357	AssertRCReturn(rc, rc);
6358	}
6359	}
6360	else
6361	{
6362	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6363	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6364	uIntrState = 0;
6365	}
6366	}
6367
6368	/*
6369	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6370	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6371	*/
6372	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6373	AssertRC(rc2);
6374
6375	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6376	return rc;
6377	}
6378
6379
6380	/**
6381	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6382	*
6383	* @param pVCpu Pointer to the VMCPU.
6384	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6385	* out-of-sync. Make sure to update the required fields
6386	* before using them.
6387	*/
6388	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6389	{
6390	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6391	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6392	}
6393
6394
6395	/**
6396	* Injects a double-fault (#DF) exception into the VM.
6397	*
6398	* @returns VBox status code (informational status code included).
6399	* @param pVCpu Pointer to the VMCPU.
6400	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6401	* out-of-sync. Make sure to update the required fields
6402	* before using them.
6403	*/
6404	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6405	{
6406	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6407	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6408	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6409	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6410	puIntrState);
6411	}
6412
6413
6414	/**
6415	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6416	*
6417	* @param pVCpu Pointer to the VMCPU.
6418	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6419	* out-of-sync. Make sure to update the required fields
6420	* before using them.
6421	*/
6422	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6423	{
6424	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6425	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6426	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6427	}
6428
6429
6430	/**
6431	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6432	*
6433	* @param pVCpu Pointer to the VMCPU.
6434	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6435	* out-of-sync. Make sure to update the required fields
6436	* before using them.
6437	* @param cbInstr The value of RIP that is to be pushed on the guest
6438	* stack.
6439	*/
6440	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6441	{
6442	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6443	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6444	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6445	}
6446
6447
6448	/**
6449	* Injects a general-protection (#GP) fault into the VM.
6450	*
6451	* @returns VBox status code (informational status code included).
6452	* @param pVCpu Pointer to the VMCPU.
6453	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6454	* out-of-sync. Make sure to update the required fields
6455	* before using them.
6456	* @param u32ErrorCode The error code associated with the #GP.
6457	*/
6458	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6459	uint32_t *puIntrState)
6460	{
6461	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6462	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6463	if (fErrorCodeValid)
6464	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6465	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6466	puIntrState);
6467	}
6468
6469
6470	/**
6471	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6472	*
6473	* @param pVCpu Pointer to the VMCPU.
6474	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6475	* out-of-sync. Make sure to update the required fields
6476	* before using them.
6477	* @param uVector The software interrupt vector number.
6478	* @param cbInstr The value of RIP that is to be pushed on the guest
6479	* stack.
6480	*/
6481	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6482	{
6483	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6484	if ( uVector == X86_XCPT_BP
6485	\|\| uVector == X86_XCPT_OF)
6486	{
6487	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6488	}
6489	else
6490	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6491	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6492	}
6493
6494
6495	/**
6496	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6497	* stack.
6498	*
6499	* @returns VBox status code (information status code included).
6500	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6501	* @param pVM Pointer to the VM.
6502	* @param pMixedCtx Pointer to the guest-CPU context.
6503	* @param uValue The value to push to the guest stack.
6504	*/
6505	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6506	{
6507	/*
6508	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6509	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6510	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6511	*/
6512	if (pMixedCtx->sp == 1)
6513	return VINF_EM_RESET;
6514	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6515	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6516	AssertRCReturn(rc, rc);
6517	return rc;
6518	}
6519
6520
6521	/**
6522	* Injects an event into the guest upon VM-entry by updating the relevant fields
6523	* in the VM-entry area in the VMCS.
6524	*
6525	* @returns VBox status code (informational error codes included).
6526	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6527	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6528	*
6529	* @param pVCpu Pointer to the VMCPU.
6530	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6531	* be out-of-sync. Make sure to update the required
6532	* fields before using them.
6533	* @param u64IntrInfo The VM-entry interruption-information field.
6534	* @param cbInstr The VM-entry instruction length in bytes (for
6535	* software interrupts, exceptions and privileged
6536	* software exceptions).
6537	* @param u32ErrCode The VM-entry exception error code.
6538	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6539	* @param puIntrState Pointer to the current guest interruptibility-state.
6540	* This interruptibility-state will be updated if
6541	* necessary. This cannot not be NULL.
6542	*
6543	* @remarks No-long-jump zone!!!
6544	* @remarks Requires CR0!
6545	*/
6546	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6547	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6548	{
6549	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6550	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6551	Assert(puIntrState);
6552	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6553
6554	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6555	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6556
6557	#ifdef VBOX_STRICT
6558	/* Validate the error-code-valid bit for hardware exceptions. */
6559	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6560	{
6561	switch (uVector)
6562	{
6563	case X86_XCPT_PF:
6564	case X86_XCPT_DF:
6565	case X86_XCPT_TS:
6566	case X86_XCPT_NP:
6567	case X86_XCPT_SS:
6568	case X86_XCPT_GP:
6569	case X86_XCPT_AC:
6570	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6571	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6572	/* fallthru */
6573	default:
6574	break;
6575	}
6576	}
6577	#endif
6578
6579	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6580	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6581	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6582
6583	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6584
6585	/* We require CR0 to check if the guest is in real-mode. */
6586	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6587	AssertRCReturn(rc, rc);
6588
6589	/*
6590	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6591	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6592	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6593	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6594	*/
6595	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6596	{
6597	PVM pVM = pVCpu->CTX_SUFF(pVM);
6598	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6599	{
6600	Assert(PDMVmmDevHeapIsEnabled(pVM));
6601	Assert(pVM->hm.s.vmx.pRealModeTSS);
6602
6603	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6604	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6605	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6606	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6607	AssertRCReturn(rc, rc);
6608	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6609
6610	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6611	const size_t cbIdtEntry = 4;
6612	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6613	{
6614	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6615	if (uVector == X86_XCPT_DF)
6616	return VINF_EM_RESET;
6617	else if (uVector == X86_XCPT_GP)
6618	{
6619	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6620	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6621	}
6622
6623	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6624	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6625	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6626	}
6627
6628	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6629	uint16_t uGuestIp = pMixedCtx->ip;
6630	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6631	{
6632	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6633	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6634	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6635	}
6636	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6637	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6638
6639	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6640	uint16_t offIdtEntry = 0;
6641	RTSEL selIdtEntry = 0;
6642	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6643	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6644	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6645	AssertRCReturn(rc, rc);
6646
6647	/* Construct the stack frame for the interrupt/exception handler. */
6648	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6649	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6650	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6651	AssertRCReturn(rc, rc);
6652
6653	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6654	if (rc == VINF_SUCCESS)
6655	{
6656	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6657	pMixedCtx->rip = offIdtEntry;
6658	pMixedCtx->cs.Sel = selIdtEntry;
6659	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6660	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6661	&& uVector == X86_XCPT_PF)
6662	{
6663	pMixedCtx->cr2 = GCPtrFaultAddress;
6664	}
6665	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6666	\| HM_CHANGED_GUEST_RIP
6667	\| HM_CHANGED_GUEST_RFLAGS
6668	\| HM_CHANGED_GUEST_RSP;
6669
6670	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6671	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6672	{
6673	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6674	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6675	Log4(("Clearing inhibition due to STI.\n"));
6676	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6677	}
6678	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6679	}
6680	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6681	return rc;
6682	}
6683	else
6684	{
6685	/*
6686	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6687	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6688	*/
6689	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6690	}
6691	}
6692
6693	/* Validate. */
6694	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6695	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6696	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6697
6698	/* Inject. */
6699	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6700	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6701	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6702	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6703
6704	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6705	&& uVector == X86_XCPT_PF)
6706	{
6707	pMixedCtx->cr2 = GCPtrFaultAddress;
6708	}
6709
6710	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6711	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6712
6713	AssertRCReturn(rc, rc);
6714	return rc;
6715	}
6716
6717
6718	/**
6719	* Enters the VT-x session.
6720	*
6721	* @returns VBox status code.
6722	* @param pVM Pointer to the VM.
6723	* @param pVCpu Pointer to the VMCPU.
6724	* @param pCpu Pointer to the CPU info struct.
6725	*/
6726	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6727	{
6728	AssertPtr(pVM);
6729	AssertPtr(pVCpu);
6730	Assert(pVM->hm.s.vmx.fSupported);
6731	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6732	NOREF(pCpu);
6733
6734	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6735
6736	#ifdef VBOX_STRICT
6737	/* Make sure we're in VMX root mode. */
6738	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6739	if (!(u32HostCR4 & X86_CR4_VMXE))
6740	{
6741	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6742	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6743	}
6744	#endif
6745
6746	/* Load the active VMCS as the current one. */
6747	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6748	if (RT_FAILURE(rc))
6749	return rc;
6750
6751	/** @todo this will change with preemption hooks where can VMRESUME as long
6752	* as we're no preempted. */
6753	pVCpu->hm.s.fResumeVM = false;
6754	return VINF_SUCCESS;
6755	}
6756
6757
6758	/**
6759	* Leaves the VT-x session.
6760	*
6761	* @returns VBox status code.
6762	* @param pVM Pointer to the VM.
6763	* @param pVCpu Pointer to the VMCPU.
6764	* @param pCtx Pointer to the guest-CPU context.
6765	*/
6766	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6767	{
6768	AssertPtr(pVCpu);
6769	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6770	NOREF(pVM);
6771	NOREF(pCtx);
6772
6773	/** @todo this will change with preemption hooks where we only VMCLEAR when
6774	* we are actually going to be preempted, not all the time like we
6775	* currently do. */
6776
6777	/* Restore host-state bits that VT-x only restores partially. */
6778	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6779	{
6780	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6781	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6782	}
6783
6784	/*
6785	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6786	* and mark the VMCS launch-state as "clear".
6787	*/
6788	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6789	return rc;
6790	}
6791
6792
6793	/**
6794	* Saves the host state in the VMCS host-state.
6795	* Sets up the VM-exit MSR-load area.
6796	*
6797	* The CPU state will be loaded from these fields on every successful VM-exit.
6798	*
6799	* @returns VBox status code.
6800	* @param pVM Pointer to the VM.
6801	* @param pVCpu Pointer to the VMCPU.
6802	*
6803	* @remarks No-long-jump zone!!!
6804	*/
6805	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6806	{
6807	AssertPtr(pVM);
6808	AssertPtr(pVCpu);
6809	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6810
6811	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6812
6813	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6814	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6815	return VINF_SUCCESS;
6816
6817	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6818	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6819
6820	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6821	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6822
6823	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6824	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6825
6826	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6827	return rc;
6828	}
6829
6830
6831	/**
6832	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6833	* loaded from these fields on every successful VM-entry.
6834	*
6835	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6836	* Sets up the VM-entry controls.
6837	* Sets up the appropriate VMX non-root function to execute guest code based on
6838	* the guest CPU mode.
6839	*
6840	* @returns VBox status code.
6841	* @param pVM Pointer to the VM.
6842	* @param pVCpu Pointer to the VMCPU.
6843	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6844	* out-of-sync. Make sure to update the required fields
6845	* before using them.
6846	*
6847	* @remarks No-long-jump zone!!!
6848	*/
6849	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6850	{
6851	AssertPtr(pVM);
6852	AssertPtr(pVCpu);
6853	AssertPtr(pMixedCtx);
6854	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6855
6856	#ifdef LOG_ENABLED
6857	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
6858	* probably not initialized yet? Anyway this will do for now. */
6859	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
6860	VMMR0LogFlushDisable(pVCpu);
6861	#endif
6862
6863	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6864
6865	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6866
6867	/* Determine real-on-v86 mode. */
6868	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6869	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6870	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6871	{
6872	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6873	}
6874
6875	/*
6876	* Load the guest-state into the VMCS.
6877	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6878	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6879	*/
6880	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6881	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6882
6883	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6884	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6885
6886	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6887	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6888
6889	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6890	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6891
6892	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6893	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6894	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6895
6896	rc = hmR0VmxLoadGuestDebugState(pVCpu, pMixedCtx);
6897	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugState: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6898
6899	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6900	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6901
6902	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6903	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6904
6905	/* Must be done after hmR0VmxLoadGuestDebugState() as it may have updated eflags.TF for debugging purposes. */
6906	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6907	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6908
6909	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6910	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6911
6912	/* Clear any unused and reserved bits. */
6913	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
6914
6915	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6916	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
6917	pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
6918
6919	#ifdef LOG_ENABLED
6920	/* Only reenable log-flushing if the caller has it enabled. */
6921	if (!fCallerDisabledLogFlush)
6922	VMMR0LogFlushEnable(pVCpu);
6923	#endif
6924
6925	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6926	return rc;
6927	}
6928
6929
6930	/**
6931	* Loads the guest state into the VMCS guest-state area.
6932	*
6933	* @returns VBox status code.
6934	* @param pVM Pointer to the VM.
6935	* @param pVCpu Pointer to the VMCPU.
6936	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6937	* out-of-sync. Make sure to update the required fields
6938	* before using them.
6939	*
6940	* @remarks No-long-jump zone!!!
6941	*/
6942	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6943	{
6944	/*
6945	* Avoid reloading the guest state on longjmp reentrants and do it lazily just before executing the guest.
6946	* This only helps when we get rescheduled more than once to a different host CPU on a longjmp trip before
6947	* finally executing guest code.
6948	*/
6949	return VINF_SUCCESS;
6950	}
6951
6952
6953	/**
6954	* Does the preparations before executing guest code in VT-x.
6955	*
6956	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6957	* recompiler. We must be cautious what we do here regarding committing
6958	* guest-state information into the VMCS assuming we assuredly execute the
6959	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
6960	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6961	* that the recompiler can (and should) use them when it resumes guest
6962	* execution. Otherwise such operations must be done when we can no longer
6963	* exit to ring-3.
6964	*
6965	* @returns VBox status code (informational status codes included).
6966	* @retval VINF_SUCCESS if we can proceed with running the guest.
6967	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6968	* into the guest.
6969	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6970	*
6971	* @param pVM Pointer to the VM.
6972	* @param pVCpu Pointer to the VMCPU.
6973	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6974	* out-of-sync. Make sure to update the required fields
6975	* before using them.
6976	* @param pVmxTransient Pointer to the VMX transient structure.
6977	*
6978	* @remarks Called with preemption disabled.
6979	*/
6980	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6981	{
6982	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6983
6984	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6985	PGMRZDynMapFlushAutoSet(pVCpu);
6986	#endif
6987
6988	/* Check force flag actions that might require us to go back to ring-3. */
6989	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6990	if (rc != VINF_SUCCESS)
6991	return rc;
6992
6993	#ifndef IEM_VERIFICATION_MODE_FULL
6994	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6995	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6996	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6997	{
6998	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6999	RTGCPHYS GCPhysApicBase;
7000	GCPhysApicBase = pMixedCtx->msrApicBase;
7001	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7002
7003	/* Unalias any existing mapping. */
7004	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7005	AssertRCReturn(rc, rc);
7006
7007	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7008	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7009	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7010	AssertRCReturn(rc, rc);
7011
7012	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7013	}
7014	#endif /* !IEM_VERIFICATION_MODE_FULL */
7015
7016	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7017	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
7018	pVmxTransient->uEFlags = ASMIntDisableFlags();
7019	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7020	{
7021	ASMSetFlags(pVmxTransient->uEFlags);
7022	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7023	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
7024	return VINF_EM_RAW_INTERRUPT;
7025	}
7026	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7027	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7028	#endif
7029
7030	/*
7031	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
7032	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
7033	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
7034	*/
7035	/** @todo Rework event evaluation and injection to be completely separate. */
7036	if (TRPMHasTrap(pVCpu))
7037	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7038
7039	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7040	AssertRCReturn(rc, rc);
7041	return rc;
7042	}
7043
7044
7045	/**
7046	* Prepares to run guest code in VT-x and we've committed to doing so. This
7047	* means there is no backing out to ring-3 or anywhere else at this
7048	* point.
7049	*
7050	* @param pVM Pointer to the VM.
7051	* @param pVCpu Pointer to the VMCPU.
7052	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7053	* out-of-sync. Make sure to update the required fields
7054	* before using them.
7055	* @param pVmxTransient Pointer to the VMX transient structure.
7056	*
7057	* @remarks Called with preemption disabled.
7058	* @remarks No-long-jump zone!!!
7059	*/
7060	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7061	{
7062	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7063	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7064
7065	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7066	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
7067	pVmxTransient->uEFlags = ASMIntDisableFlags();
7068	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7069	#endif
7070
7071	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
7072	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7073	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7074	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7075	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
7076	#endif
7077	int rc = VINF_SUCCESS;
7078	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7079	{
7080	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7081	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7082	}
7083	else if (pVCpu->hm.s.fContextUseFlags)
7084	{
7085	rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7086	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7087	}
7088	AssertRC(rc);
7089	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
7090
7091	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7092	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7093	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7094	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7095	#endif
7096
7097	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
7098	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7099	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7100
7101	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7102	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
7103	{
7104	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7105	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7106	}
7107
7108	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7109	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
7110	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
7111
7112	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7113
7114	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7115	to start executing. */
7116
7117	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7118	/*
7119	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7120	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7121	*/
7122	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7123	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7124	{
7125	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7126	uint64_t u64HostTscAux = 0;
7127	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7128	AssertRC(rc2);
7129	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7130	}
7131	#endif
7132	}
7133
7134
7135	/**
7136	* Performs some essential restoration of state after running guest code in
7137	* VT-x.
7138	*
7139	* @param pVM Pointer to the VM.
7140	* @param pVCpu Pointer to the VMCPU.
7141	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7142	* out-of-sync. Make sure to update the required fields
7143	* before using them.
7144	* @param pVmxTransient Pointer to the VMX transient structure.
7145	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7146	*
7147	* @remarks Called with interrupts disabled.
7148	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7149	* unconditionally when it is safe to do so.
7150	*/
7151	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7152	{
7153	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7154
7155	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7156	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7157	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7158	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7159	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7160
7161	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7162	{
7163	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7164	/* Restore host's TSC_AUX. */
7165	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7166	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7167	#endif
7168	/** @todo Find a way to fix hardcoding a guestimate. */
7169	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7170	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7171	}
7172
7173	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7174	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7175	Assert(!(ASMGetFlags() & X86_EFL_IF));
7176	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7177
7178	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
7179	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7180
7181	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7182	uint32_t uExitReason;
7183	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7184	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7185	AssertRC(rc);
7186	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7187	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7188
7189	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
7190	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7191
7192	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7193	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7194	{
7195	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7196	pVmxTransient->fVMEntryFailed));
7197	return;
7198	}
7199
7200	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7201	{
7202	/* Update the guest interruptibility-state from the VMCS. */
7203	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7204	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7205	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7206	AssertRC(rc);
7207	#endif
7208	/*
7209	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7210	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7211	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7212	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7213	*/
7214	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7215	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7216	{
7217	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7218	AssertRC(rc);
7219	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7220	}
7221	}
7222	}
7223
7224
7225	/**
7226	* Runs the guest code using VT-x.
7227	*
7228	* @returns VBox status code.
7229	* @param pVM Pointer to the VM.
7230	* @param pVCpu Pointer to the VMCPU.
7231	* @param pCtx Pointer to the guest-CPU context.
7232	*
7233	* @remarks Called with preemption disabled.
7234	*/
7235	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7236	{
7237	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7238	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7239
7240	VMXTRANSIENT VmxTransient;
7241	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7242	int rc = VERR_INTERNAL_ERROR_5;
7243	uint32_t cLoops = 0;
7244
7245	for (;; cLoops++)
7246	{
7247	Assert(!HMR0SuspendPending());
7248	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7249	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7250	(unsigned)RTMpCpuId(), cLoops));
7251
7252	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7253	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7254	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7255	if (rc != VINF_SUCCESS)
7256	break;
7257
7258	/*
7259	* No longjmps to ring-3 from this point on!!!
7260	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7261	* This also disables flushing of the R0-logger instance (if any).
7262	*/
7263	VMMRZCallRing3Disable(pVCpu);
7264	VMMRZCallRing3RemoveNotification(pVCpu);
7265	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7266
7267	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7268	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7269
7270	/*
7271	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7272	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7273	*/
7274	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7275	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7276	{
7277	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7278	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7279	return rc;
7280	}
7281
7282	/* Handle the VM-exit. */
7283	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7284	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7285	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7286	HMVMX_START_EXIT_DISPATCH_PROF();
7287	#ifdef HMVMX_USE_FUNCTION_TABLE
7288	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7289	#else
7290	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7291	#endif
7292	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7293	if (rc != VINF_SUCCESS)
7294	break;
7295	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7296	{
7297	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7298	rc = VINF_EM_RAW_INTERRUPT;
7299	break;
7300	}
7301	}
7302
7303	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7304	if (rc == VERR_EM_INTERPRETER)
7305	rc = VINF_EM_RAW_EMULATE_INSTR;
7306	else if (rc == VINF_EM_RESET)
7307	rc = VINF_EM_TRIPLE_FAULT;
7308	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7309	return rc;
7310	}
7311
7312
7313	#ifndef HMVMX_USE_FUNCTION_TABLE
7314	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7315	{
7316	int rc;
7317	switch (rcReason)
7318	{
7319	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7320	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7321	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7322	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7323	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7324	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7325	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7326	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7327	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7328	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7329	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7330	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7331	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7332	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7333	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7334	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7335	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7336	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7337	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7338	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7339	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7340	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7341	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7342	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7343	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7344	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7345	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7346	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7347	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7348	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7349	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7350	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7351	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7352
7353	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7354	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7355	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7356	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7357	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7358	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7359	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7360	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7361	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7362
7363	case VMX_EXIT_VMCALL:
7364	case VMX_EXIT_VMCLEAR:
7365	case VMX_EXIT_VMLAUNCH:
7366	case VMX_EXIT_VMPTRLD:
7367	case VMX_EXIT_VMPTRST:
7368	case VMX_EXIT_VMREAD:
7369	case VMX_EXIT_VMRESUME:
7370	case VMX_EXIT_VMWRITE:
7371	case VMX_EXIT_VMXOFF:
7372	case VMX_EXIT_VMXON:
7373	case VMX_EXIT_INVEPT:
7374	case VMX_EXIT_INVVPID:
7375	case VMX_EXIT_VMFUNC:
7376	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7377	break;
7378	default:
7379	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7380	break;
7381	}
7382	return rc;
7383	}
7384	#endif
7385
7386	#ifdef DEBUG
7387	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7388	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7389	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7390
7391	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7392	do \
7393	{ \
7394	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7395	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7396	} while (0)
7397
7398	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7399	do { \
7400	AssertPtr(pVCpu); \
7401	AssertPtr(pMixedCtx); \
7402	AssertPtr(pVmxTransient); \
7403	Assert(pVmxTransient->fVMEntryFailed == false); \
7404	Assert(ASMIntAreEnabled()); \
7405	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7406	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7407	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7408	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7409	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7410	HMVMX_ASSERT_PREEMPT_CPUID(); \
7411	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7412	} while (0)
7413
7414	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7415	do { \
7416	Log4Func(("\n")); \
7417	} while(0)
7418	#else /* Release builds */
7419	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7420	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7421	#endif
7422
7423
7424	/**
7425	* Advances the guest RIP after reading it from the VMCS.
7426	*
7427	* @returns VBox status code.
7428	* @param pVCpu Pointer to the VMCPU.
7429	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7430	* out-of-sync. Make sure to update the required fields
7431	* before using them.
7432	* @param pVmxTransient Pointer to the VMX transient structure.
7433	*
7434	* @remarks No-long-jump zone!!!
7435	*/
7436	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7437	{
7438	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7439	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7440	AssertRCReturn(rc, rc);
7441
7442	pMixedCtx->rip += pVmxTransient->cbInstr;
7443	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7444	return rc;
7445	}
7446
7447
7448	/**
7449	* Tries to determine what part of the guest-state VT-x has deemed as invalid
7450	* and update error record fields accordingly.
7451	*
7452	* @return VMX_IGS_* return codes.
7453	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7454	* wrong with the guest state.
7455	*
7456	* @param pVM Pointer to the VM.
7457	* @param pVCpu Pointer to the VMCPU.
7458	* @param pCtx Pointer to the guest-CPU state.
7459	*/
7460	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7461	{
7462	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7463	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7464	uError = (err); \
7465	break; \
7466	} else do {} while (0)
7467	/* Duplicate of IEM_IS_CANONICAL(). */
7468	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7469
7470	int rc;
7471	uint64_t u64Val;
7472	uint32_t u32Val;
7473	uint32_t uError = VMX_IGS_ERROR;
7474	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7475
7476	do
7477	{
7478	/*
7479	* CR0.
7480	*/
7481	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7482	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7483	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7484	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7485	if (fUnrestrictedGuest)
7486	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
7487
7488	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7489	AssertRCBreak(rc);
7490	HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
7491	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
7492	if ( !fUnrestrictedGuest
7493	&& (u32Val & X86_CR0_PG)
7494	&& !(u32Val & X86_CR0_PE))
7495	{
7496	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7497	}
7498
7499	/*
7500	* CR4.
7501	*/
7502	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7503	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7504	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7505	AssertRCBreak(rc);
7506	HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
7507	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
7508
7509	/*
7510	* IA32_DEBUGCTL MSR.
7511	*/
7512	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7513	AssertRCBreak(rc);
7514	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7515	&& (u64Val & 0xfffffe3c)) /* Bits 31-9, bits 2-5 MBZ. */
7516	{
7517	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7518	}
7519	uint64_t u64DebugCtlMsr = u64Val;
7520
7521	#ifdef VBOX_STRICT
7522	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
7523	AssertRCBreak(rc);
7524	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
7525	#endif
7526	const bool fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
7527
7528	/*
7529	* RIP and RFLAGS.
7530	*/
7531	uint32_t u32EFlags;
7532	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7533	if (HMVMX_IS_64BIT_HOST_MODE())
7534	{
7535	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
7536	AssertRCBreak(rc);
7537	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
7538	if ( !fLongModeGuest
7539	\|\| !pCtx->cs.Attr.n.u1Long)
7540	{
7541	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
7542	}
7543	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
7544	* must be identical if the "IA32e mode guest" VM-entry control is 1
7545	* and CS.L is 1. No check applies if the CPU supports 64
7546	* linear-address bits. */
7547
7548	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
7549	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7550	AssertRCBreak(rc);
7551	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
7552	VMX_IGS_RFLAGS_RESERVED);
7553	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7554	u32EFlags = u64Val;
7555	}
7556	else
7557	#endif
7558	{
7559	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32EFlags);
7560	AssertRCBreak(rc);
7561	HMVMX_CHECK_BREAK(!(u32EFlags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
7562	HMVMX_CHECK_BREAK((u32EFlags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7563	}
7564
7565	if ( fLongModeGuest
7566	\|\| !(pCtx->cr0 & X86_CR0_PE))
7567	{
7568	HMVMX_CHECK_BREAK(!(u32EFlags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
7569	}
7570
7571	uint32_t u32EntryInfo;
7572	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7573	AssertRCBreak(rc);
7574	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
7575	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7576	{
7577	HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
7578	}
7579
7580	/*
7581	* 64-bit checks.
7582	*/
7583	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7584	if (HMVMX_IS_64BIT_HOST_MODE())
7585	{
7586	if ( fLongModeGuest
7587	&& !fUnrestrictedGuest)
7588	{
7589	HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
7590	HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
7591	}
7592
7593	if ( !fLongModeGuest
7594	&& (pCtx->cr4 & X86_CR4_PCIDE))
7595	{
7596	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
7597	}
7598
7599	/** @todo CR3 field must be such that bits 63:52 and bits in the range
7600	* 51:32 beyond the processor's physical-address width are 0. */
7601
7602	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7603	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
7604	{
7605	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
7606	}
7607
7608	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
7609	AssertRCBreak(rc);
7610	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
7611
7612	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
7613	AssertRCBreak(rc);
7614	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
7615	}
7616	#endif
7617
7618	/*
7619	* PERF_GLOBAL MSR.
7620	*/
7621	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
7622	{
7623	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
7624	AssertRCBreak(rc);
7625	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
7626	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63-35, bits 31-2 MBZ. */
7627	}
7628
7629	/*
7630	* PAT MSR.
7631	*/
7632	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
7633	{
7634	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
7635	AssertRCBreak(rc);
7636	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
7637	for (unsigned i = 0; i < 8; i++)
7638	{
7639	uint8_t u8Val = (u64Val & 0x7);
7640	if ( u8Val != 0 /* UC */
7641	\|\| u8Val != 1 /* WC */
7642	\|\| u8Val != 4 /* WT */
7643	\|\| u8Val != 5 /* WP */
7644	\|\| u8Val != 6 /* WB */
7645	\|\| u8Val != 7 /* UC- */)
7646	{
7647	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
7648	}
7649	u64Val >>= 3;
7650	}
7651	}
7652
7653	/*
7654	* EFER MSR.
7655	*/
7656	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
7657	{
7658	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
7659	AssertRCBreak(rc);
7660	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
7661	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63-12, bit 9, bits 7-1 MBZ. */
7662	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
7663	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
7664	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7665	\|\| (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
7666	}
7667
7668	/*
7669	* Segment registers.
7670	*/
7671	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7672	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
7673	if (!(u32EFlags & X86_EFL_VM))
7674	{
7675	/* CS */
7676	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
7677	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
7678	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
7679	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
7680	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7681	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
7682	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7683	/* CS cannot be loaded with NULL in protected mode. */
7684	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
7685	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
7686	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
7687	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
7688	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
7689	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
7690	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
7691	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
7692	else
7693	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
7694
7695	/* SS */
7696	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7697	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
7698	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
7699	if ( !(pCtx->cr0 & X86_CR0_PE)
7700	\|\| pCtx->cs.Attr.n.u4Type == 3)
7701	{
7702	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
7703	}
7704	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
7705	{
7706	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
7707	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
7708	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
7709	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
7710	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
7711	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7712	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
7713	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7714	}
7715
7716	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
7717	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
7718	{
7719	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
7720	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
7721	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7722	\|\| pCtx->ds.Attr.n.u4Type > 11
7723	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7724	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
7725	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
7726	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
7727	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7728	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
7729	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7730	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7731	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
7732	}
7733	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
7734	{
7735	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
7736	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
7737	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7738	\|\| pCtx->es.Attr.n.u4Type > 11
7739	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7740	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
7741	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
7742	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
7743	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7744	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
7745	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7746	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7747	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
7748	}
7749	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
7750	{
7751	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
7752	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
7753	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7754	\|\| pCtx->fs.Attr.n.u4Type > 11
7755	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
7756	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
7757	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
7758	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
7759	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7760	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
7761	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7762	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7763	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
7764	}
7765	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
7766	{
7767	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
7768	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
7769	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7770	\|\| pCtx->gs.Attr.n.u4Type > 11
7771	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
7772	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
7773	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
7774	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
7775	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
7776	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
7777	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
7778	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7779	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
7780	}
7781	/* 64-bit capable CPUs. */
7782	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7783	if (HMVMX_IS_64BIT_HOST_MODE())
7784	{
7785	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
7786	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
7787	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7788	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
7789	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
7790	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
7791	VMX_IGS_LONGMODE_SS_BASE_INVALID);
7792	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
7793	VMX_IGS_LONGMODE_DS_BASE_INVALID);
7794	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
7795	VMX_IGS_LONGMODE_ES_BASE_INVALID);
7796	}
7797	#endif
7798	}
7799	else
7800	{
7801	/* V86 mode checks. */
7802	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
7803	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7804	{
7805	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
7806	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
7807	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
7808	}
7809	else
7810	{
7811	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
7812	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
7813	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
7814	}
7815
7816	/* CS */
7817	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
7818	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
7819	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
7820	/* SS */
7821	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
7822	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
7823	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
7824	/* DS */
7825	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
7826	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
7827	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
7828	/* ES */
7829	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
7830	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
7831	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
7832	/* FS */
7833	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
7834	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
7835	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
7836	/* GS */
7837	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
7838	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
7839	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
7840	/* 64-bit capable CPUs. */
7841	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7842	if (HMVMX_IS_64BIT_HOST_MODE())
7843	{
7844	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
7845	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
7846	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7847	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
7848	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
7849	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
7850	VMX_IGS_LONGMODE_SS_BASE_INVALID);
7851	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
7852	VMX_IGS_LONGMODE_DS_BASE_INVALID);
7853	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
7854	VMX_IGS_LONGMODE_ES_BASE_INVALID);
7855	}
7856	#endif
7857	}
7858
7859	/*
7860	* TR.
7861	*/
7862	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
7863	/* 64-bit capable CPUs. */
7864	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7865	if (HMVMX_IS_64BIT_HOST_MODE())
7866	{
7867	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
7868	}
7869	#endif
7870	if (fLongModeGuest)
7871	{
7872	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
7873	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
7874	}
7875	else
7876	{
7877	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
7878	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
7879	VMX_IGS_TR_ATTR_TYPE_INVALID);
7880	}
7881	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
7882	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
7883	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11-8 MBZ. */
7884	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
7885	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
7886	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
7887	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
7888	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
7889
7890	/*
7891	* GDTR and IDTR.
7892	*/
7893	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7894	if (HMVMX_IS_64BIT_HOST_MODE())
7895	{
7896	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7897	AssertRCBreak(rc);
7898	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
7899
7900	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7901	AssertRCBreak(rc);
7902	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
7903	}
7904	#endif
7905
7906	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7907	AssertRCBreak(rc);
7908	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
7909
7910	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7911	AssertRCBreak(rc);
7912	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
7913
7914	/*
7915	* Guest Non-Register State.
7916	*/
7917	/* Activity State. */
7918	uint32_t u32ActivityState;
7919	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
7920	AssertRCBreak(rc);
7921	HMVMX_CHECK_BREAK( !u32ActivityState
7922	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.msr.vmx_misc)),
7923	VMX_IGS_ACTIVITY_STATE_INVALID);
7924	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
7925	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
7926	uint32_t u32IntrState;
7927	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
7928	AssertRCBreak(rc);
7929	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
7930	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7931	{
7932	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
7933	}
7934
7935	/** @todo Activity state and injecting interrupts. Left as a todo since we
7936	* currently don't use activity states but ACTIVE. */
7937
7938	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
7939	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
7940
7941	/* Guest interruptibility-state. */
7942	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
7943	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
7944	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
7945	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
7946	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
7947	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
7948	HMVMX_CHECK_BREAK( (u32EFlags & X86_EFL_IF)
7949	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
7950	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
7951	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
7952	{
7953	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7954	{
7955	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7956	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
7957	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
7958	}
7959	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7960	{
7961	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
7962	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
7963	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
7964	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
7965	}
7966	}
7967	/** @todo Assumes the processor is not in SMM. */
7968	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
7969	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
7970	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
7971	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
7972	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
7973	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
7974	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
7975	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7976	{
7977	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
7978	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
7979	}
7980
7981	/* Pending debug exceptions. */
7982	if (HMVMX_IS_64BIT_HOST_MODE())
7983	{
7984	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
7985	AssertRCBreak(rc);
7986	/* Bits 63-15, Bit 13, Bits 11-4 MBZ. */
7987	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
7988	u32Val = u64Val; /* For pending debug exceptions checks below. */
7989	}
7990	else
7991	{
7992	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
7993	AssertRCBreak(rc);
7994	/* Bits 31-15, Bit 13, Bits 11-4 MBZ. */
7995	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
7996	}
7997
7998	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7999	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8000	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8001	{
8002	if ( (u32EFlags & X86_EFL_TF)
8003	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8004	{
8005	/* Bit 14 is PendingDebug.BS. */
8006	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8007	}
8008	if ( !(u32EFlags & X86_EFL_TF)
8009	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8010	{
8011	/* Bit 14 is PendingDebug.BS. */
8012	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8013	}
8014	}
8015
8016	/* VMCS link pointer. */
8017	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8018	AssertRCBreak(rc);
8019	if (u64Val != UINT64_C(0xffffffffffffffff))
8020	{
8021	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8022	/** @todo Bits beyond the processor's physical-address width MBZ. */
8023	/** @todo 32-bit located in memory referenced by value of this field (as a
8024	* physical address) must contain the processor's VMCS revision ID. */
8025	/** @todo SMM checks. */
8026	}
8027
8028	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8029
8030	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8031	if (uError == VMX_IGS_ERROR)
8032	uError = VMX_IGS_REASON_NOT_FOUND;
8033	} while (0);
8034
8035	pVCpu->hm.s.u32HMError = uError;
8036	return uError;
8037
8038	#undef HMVMX_ERROR_BREAK
8039	#undef HMVMX_CHECK_BREAK
8040	#undef HMVMX_IS_CANONICAL
8041	}
8042
8043	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8044	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8045	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8046
8047	/** @name VM-exit handlers.
8048	* @{
8049	*/
8050
8051	/**
8052	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8053	*/
8054	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8055	{
8056	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8057	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8058	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8059	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
8060	Assert(ASMIntAreEnabled());
8061	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8062	return VINF_SUCCESS;
8063	#endif
8064	return VINF_EM_RAW_INTERRUPT;
8065	}
8066
8067
8068	/**
8069	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8070	*/
8071	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8072	{
8073	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8074	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8075
8076	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8077	AssertRCReturn(rc, rc);
8078
8079	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8080	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8081	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8082	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8083
8084	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8085	{
8086	/*
8087	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8088	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8089	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8090	*
8091	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8092	*/
8093	VMXDispatchHostNmi();
8094	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
8095	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8096	return VINF_SUCCESS;
8097	}
8098
8099	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8100	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8101	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8102	{
8103	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8104	return VINF_SUCCESS;
8105	}
8106	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8107	{
8108	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8109	return rc;
8110	}
8111
8112	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8113	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8114	switch (uIntrType)
8115	{
8116	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8117	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8118	/* no break */
8119	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8120	{
8121	switch (uVector)
8122	{
8123	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8124	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8125	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8126	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8127	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8128	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8129	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8130	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8131	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8132	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8133	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8134	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8135	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8136	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8137	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8138	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8139	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8140	#endif
8141	default:
8142	{
8143	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8144	AssertRCReturn(rc, rc);
8145
8146	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8147	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8148	{
8149	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8150	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8151	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8152	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8153	AssertRCReturn(rc, rc);
8154	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8155	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8156	0 /* GCPtrFaultAddress */);
8157	AssertRCReturn(rc, rc);
8158	}
8159	else
8160	{
8161	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8162	pVCpu->hm.s.u32HMError = uVector;
8163	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8164	}
8165	break;
8166	}
8167	}
8168	break;
8169	}
8170
8171	default:
8172	{
8173	pVCpu->hm.s.u32HMError = uExitIntrInfo;
8174	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8175	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8176	break;
8177	}
8178	}
8179	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8180	return rc;
8181	}
8182
8183
8184	/**
8185	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8186	*/
8187	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8188	{
8189	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8190
8191	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8192	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8193	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8194	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8195	AssertRCReturn(rc, rc);
8196
8197	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8198	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8199	return VINF_SUCCESS;
8200	}
8201
8202
8203	/**
8204	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8205	*/
8206	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8207	{
8208	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8209	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8210	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8211	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8212	}
8213
8214
8215	/**
8216	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8217	*/
8218	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8219	{
8220	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8221	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8222	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8223	}
8224
8225
8226	/**
8227	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8228	*/
8229	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8230	{
8231	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8232	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8233	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8234	}
8235
8236
8237	/**
8238	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8239	*/
8240	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8241	{
8242	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8243	PVM pVM = pVCpu->CTX_SUFF(pVM);
8244	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8245	if (RT_LIKELY(rc == VINF_SUCCESS))
8246	{
8247	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8248	Assert(pVmxTransient->cbInstr == 2);
8249	}
8250	else
8251	{
8252	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8253	rc = VERR_EM_INTERPRETER;
8254	}
8255	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8256	return rc;
8257	}
8258
8259
8260	/**
8261	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8262	*/
8263	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8264	{
8265	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8266	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8267	AssertRCReturn(rc, rc);
8268
8269	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8270	return VINF_EM_RAW_EMULATE_INSTR;
8271
8272	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8273	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8274	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8275	}
8276
8277
8278	/**
8279	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8280	*/
8281	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8282	{
8283	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8284	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8285	AssertRCReturn(rc, rc);
8286
8287	PVM pVM = pVCpu->CTX_SUFF(pVM);
8288	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8289	if (RT_LIKELY(rc == VINF_SUCCESS))
8290	{
8291	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8292	Assert(pVmxTransient->cbInstr == 2);
8293	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8294	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8295	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8296	}
8297	else
8298	{
8299	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8300	rc = VERR_EM_INTERPRETER;
8301	}
8302	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8303	return rc;
8304	}
8305
8306
8307	/**
8308	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8309	*/
8310	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8311	{
8312	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8313	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8314	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8315	AssertRCReturn(rc, rc);
8316
8317	PVM pVM = pVCpu->CTX_SUFF(pVM);
8318	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8319	if (RT_LIKELY(rc == VINF_SUCCESS))
8320	{
8321	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8322	Assert(pVmxTransient->cbInstr == 3);
8323	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8324	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8325	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8326	}
8327	else
8328	{
8329	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8330	rc = VERR_EM_INTERPRETER;
8331	}
8332	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8333	return rc;
8334	}
8335
8336
8337	/**
8338	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8339	*/
8340	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8341	{
8342	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8343	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8344	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8345	AssertRCReturn(rc, rc);
8346
8347	PVM pVM = pVCpu->CTX_SUFF(pVM);
8348	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8349	if (RT_LIKELY(rc == VINF_SUCCESS))
8350	{
8351	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8352	Assert(pVmxTransient->cbInstr == 2);
8353	}
8354	else
8355	{
8356	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8357	rc = VERR_EM_INTERPRETER;
8358	}
8359	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8360	return rc;
8361	}
8362
8363
8364	/**
8365	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8366	*/
8367	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8368	{
8369	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8370	PVM pVM = pVCpu->CTX_SUFF(pVM);
8371	Assert(!pVM->hm.s.fNestedPaging);
8372
8373	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8374	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8375	AssertRCReturn(rc, rc);
8376
8377	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8378	rc = VBOXSTRICTRC_VAL(rc2);
8379	if (RT_LIKELY(rc == VINF_SUCCESS))
8380	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8381	else
8382	{
8383	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8384	pVmxTransient->uExitQualification, rc));
8385	}
8386	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8387	return rc;
8388	}
8389
8390
8391	/**
8392	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8393	*/
8394	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8395	{
8396	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8397	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8398	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8399	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8400	AssertRCReturn(rc, rc);
8401
8402	PVM pVM = pVCpu->CTX_SUFF(pVM);
8403	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8404	if (RT_LIKELY(rc == VINF_SUCCESS))
8405	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8406	else
8407	{
8408	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8409	rc = VERR_EM_INTERPRETER;
8410	}
8411	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8412	return rc;
8413	}
8414
8415
8416	/**
8417	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8418	*/
8419	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8420	{
8421	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8422	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8423	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8424	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8425	AssertRCReturn(rc, rc);
8426
8427	PVM pVM = pVCpu->CTX_SUFF(pVM);
8428	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8429	rc = VBOXSTRICTRC_VAL(rc2);
8430	if (RT_LIKELY( rc == VINF_SUCCESS
8431	\|\| rc == VINF_EM_HALT))
8432	{
8433	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8434	AssertRCReturn(rc3, rc3);
8435
8436	if ( rc == VINF_EM_HALT
8437	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8438	{
8439	rc = VINF_SUCCESS;
8440	}
8441	}
8442	else
8443	{
8444	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8445	rc = VERR_EM_INTERPRETER;
8446	}
8447	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
8448	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8450	return rc;
8451	}
8452
8453
8454	/**
8455	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8456	*/
8457	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8458	{
8459	/*
8460	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8461	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8462	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
8463	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8464	*/
8465	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8466	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8467	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8468	}
8469
8470
8471	/**
8472	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8473	*/
8474	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8475	{
8476	/*
8477	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8478	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8479	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
8480	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8481	*/
8482	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8483	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8484	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8485	}
8486
8487
8488	/**
8489	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
8490	*/
8491	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8492	{
8493	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
8494	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8495	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
8496	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8497	}
8498
8499
8500	/**
8501	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
8502	*/
8503	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8504	{
8505	/*
8506	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
8507	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
8508	* See Intel spec. 25.3 "Other Causes of VM-exits".
8509	*/
8510	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8511	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
8512	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8513	}
8514
8515
8516	/**
8517	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
8518	* VM-exit.
8519	*/
8520	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8521	{
8522	/*
8523	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
8524	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
8525	*
8526	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
8527	* See Intel spec. "23.8 Restrictions on VMX operation".
8528	*/
8529	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8530	return VINF_SUCCESS;
8531	}
8532
8533
8534	/**
8535	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
8536	* VM-exit.
8537	*/
8538	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8539	{
8540	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8541	return VINF_EM_RESET;
8542	}
8543
8544
8545	/**
8546	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
8547	*/
8548	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8549	{
8550	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8551	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
8552	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8553	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8554	AssertRCReturn(rc, rc);
8555
8556	pMixedCtx->rip++;
8557	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8558	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
8559	rc = VINF_SUCCESS;
8560	else
8561	rc = VINF_EM_HALT;
8562
8563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8564	return rc;
8565	}
8566
8567
8568	/**
8569	* VM-exit handler for instructions that result in a #UD exception delivered to
8570	* the guest.
8571	*/
8572	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8573	{
8574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8575	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
8576	return VINF_SUCCESS;
8577	}
8578
8579
8580	/**
8581	* VM-exit handler for expiry of the VMX preemption timer.
8582	*/
8583	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8584	{
8585	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8586
8587	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
8588	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8589
8590	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
8591	PVM pVM = pVCpu->CTX_SUFF(pVM);
8592	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
8593	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
8594	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
8595	}
8596
8597
8598	/**
8599	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
8600	*/
8601	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8602	{
8603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8604
8605	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
8606	/** @todo check if XSETBV is supported by the recompiler. */
8607	return VERR_EM_INTERPRETER;
8608	}
8609
8610
8611	/**
8612	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
8613	*/
8614	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8615	{
8616	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8617
8618	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
8619	/** @todo implement EMInterpretInvpcid() */
8620	return VERR_EM_INTERPRETER;
8621	}
8622
8623
8624	/**
8625	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
8626	* Error VM-exit.
8627	*/
8628	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8629	{
8630	uint32_t uIntrState;
8631	HMVMXHCUINTREG uHCReg;
8632	uint64_t u64Val;
8633	uint32_t u32Val;
8634
8635	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
8636	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
8637	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
8638	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
8639	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8640	AssertRCReturn(rc, rc);
8641
8642	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
8643	NOREF(uInvalidReason);
8644
8645	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
8646	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
8647	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
8648	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
8649
8650	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
8651	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
8652	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
8653	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
8654	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
8655	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
8656	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
8657	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
8658	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
8659	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
8660	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
8661	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
8662
8663	PVM pVM = pVCpu->CTX_SUFF(pVM);
8664	HMDumpRegs(pVM, pVCpu, pMixedCtx);
8665
8666	return VERR_VMX_INVALID_GUEST_STATE;
8667	}
8668
8669
8670	/**
8671	* VM-exit handler for VM-entry failure due to an MSR-load
8672	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
8673	*/
8674	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8675	{
8676	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8677	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8678	}
8679
8680
8681	/**
8682	* VM-exit handler for VM-entry failure due to a machine-check event
8683	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
8684	*/
8685	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8686	{
8687	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8688	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8689	}
8690
8691
8692	/**
8693	* VM-exit handler for all undefined reasons. Should never ever happen.. in
8694	* theory.
8695	*/
8696	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8697	{
8698	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
8699	return VERR_VMX_UNDEFINED_EXIT_CODE;
8700	}
8701
8702
8703	/**
8704	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
8705	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
8706	* Conditional VM-exit.
8707	*/
8708	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8709	{
8710	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8711
8712	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
8713	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
8714	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
8715	return VERR_EM_INTERPRETER;
8716	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8717	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8718	}
8719
8720
8721	/**
8722	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
8723	*/
8724	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8725	{
8726	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8727
8728	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
8729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
8730	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
8731	return VERR_EM_INTERPRETER;
8732	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8733	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8734	}
8735
8736
8737	/**
8738	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
8739	*/
8740	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8741	{
8742	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8743
8744	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
8745	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8746	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8747	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8748	AssertRCReturn(rc, rc);
8749
8750	PVM pVM = pVCpu->CTX_SUFF(pVM);
8751	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8752	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
8753	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
8754	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
8755
8756	if (RT_LIKELY(rc == VINF_SUCCESS))
8757	{
8758	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8759	Assert(pVmxTransient->cbInstr == 2);
8760	}
8761	return rc;
8762	}
8763
8764
8765	/**
8766	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
8767	*/
8768	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8769	{
8770	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8771	PVM pVM = pVCpu->CTX_SUFF(pVM);
8772	int rc = VINF_SUCCESS;
8773
8774	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
8775	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8776	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8777	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8778	AssertRCReturn(rc, rc);
8779	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
8780
8781	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8782	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
8783	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
8784
8785	if (RT_LIKELY(rc == VINF_SUCCESS))
8786	{
8787	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8788
8789	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
8790	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
8791	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
8792	{
8793	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
8794	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
8795	EMInterpretWrmsr() changes it. */
8796	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8797	}
8798	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
8799	{
8800	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8801	AssertRCReturn(rc, rc);
8802	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
8803	}
8804	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
8805	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8806
8807	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
8808	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8809	{
8810	switch (pMixedCtx->ecx)
8811	{
8812	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
8813	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
8814	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
8815	case MSR_K8_FS_BASE: /* no break */
8816	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
8817	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
8818	}
8819	}
8820	#ifdef VBOX_STRICT
8821	else
8822	{
8823	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
8824	switch (pMixedCtx->ecx)
8825	{
8826	case MSR_IA32_SYSENTER_CS:
8827	case MSR_IA32_SYSENTER_EIP:
8828	case MSR_IA32_SYSENTER_ESP:
8829	case MSR_K8_FS_BASE:
8830	case MSR_K8_GS_BASE:
8831	{
8832	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
8833	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8834	}
8835
8836	case MSR_K8_LSTAR:
8837	case MSR_K6_STAR:
8838	case MSR_K8_SF_MASK:
8839	case MSR_K8_TSC_AUX:
8840	case MSR_K8_KERNEL_GS_BASE:
8841	{
8842	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
8843	pMixedCtx->ecx));
8844	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8845	}
8846	}
8847	}
8848	#endif /* VBOX_STRICT */
8849	}
8850	return rc;
8851	}
8852
8853
8854	/**
8855	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8856	*/
8857	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8858	{
8859	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8860
8861	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
8862	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
8863	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
8864	return VERR_EM_INTERPRETER;
8865	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8866	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8867	}
8868
8869
8870	/**
8871	* VM-exit handler for when the TPR value is lowered below the specified
8872	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8873	*/
8874	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8875	{
8876	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8877	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8878
8879	/*
8880	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8881	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8882	* resume guest execution.
8883	*/
8884	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8885	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8886	return VINF_SUCCESS;
8887	}
8888
8889
8890	/**
8891	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8892	* VM-exit.
8893	*
8894	* @retval VINF_SUCCESS when guest execution can continue.
8895	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8896	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8897	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8898	* recompiler.
8899	*/
8900	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8901	{
8902	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8903	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8904	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8905	AssertRCReturn(rc, rc);
8906
8907	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8908	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8909	PVM pVM = pVCpu->CTX_SUFF(pVM);
8910	switch (uAccessType)
8911	{
8912	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8913	{
8914	#if 0
8915	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8916	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8917	#else
8918	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8919	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8920	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8921	#endif
8922	AssertRCReturn(rc, rc);
8923
8924	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8925	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8926	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8927	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8928
8929	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8930	{
8931	case 0: /* CR0 */
8932	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8933	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8934	break;
8935	case 2: /* C2 **/
8936	/* Nothing to do here, CR2 it's not part of the VMCS. */
8937	break;
8938	case 3: /* CR3 */
8939	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8940	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8941	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8942	break;
8943	case 4: /* CR4 */
8944	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8945	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8946	break;
8947	case 8: /* CR8 */
8948	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8949	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8950	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8951	break;
8952	default:
8953	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8954	break;
8955	}
8956
8957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8958	break;
8959	}
8960
8961	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8962	{
8963	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8964	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8965	AssertRCReturn(rc, rc);
8966	Assert( !pVM->hm.s.fNestedPaging
8967	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8968	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8969
8970	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8971	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8972	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8973
8974	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8975	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8976	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8977	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8978	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8979	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8980	break;
8981	}
8982
8983	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8984	{
8985	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8986	AssertRCReturn(rc, rc);
8987	rc = EMInterpretCLTS(pVM, pVCpu);
8988	AssertRCReturn(rc, rc);
8989	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8990	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8991	Log4(("CRX CLTS write rc=%d\n", rc));
8992	break;
8993	}
8994
8995	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8996	{
8997	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8998	AssertRCReturn(rc, rc);
8999	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9000	if (RT_LIKELY(rc == VINF_SUCCESS))
9001	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9002	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9003	Log4(("CRX LMSW write rc=%d\n", rc));
9004	break;
9005	}
9006
9007	default:
9008	{
9009	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9010	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9011	}
9012	}
9013
9014	/* Validate possible error codes. */
9015	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9016	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9017	if (RT_SUCCESS(rc))
9018	{
9019	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9020	AssertRCReturn(rc2, rc2);
9021	}
9022
9023	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9024	return rc;
9025	}
9026
9027
9028	/**
9029	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9030	* VM-exit.
9031	*/
9032	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9033	{
9034	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9035	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9036
9037	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9038	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9039	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9040	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9041	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9042	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9043	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9044	AssertRCReturn(rc2, rc2);
9045
9046	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9047	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9048	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9049	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9050	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9051	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9052	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9053
9054	/* I/O operation lookup arrays. */
9055	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9056	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9057
9058	VBOXSTRICTRC rcStrict;
9059	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9060	const uint32_t cbInstr = pVmxTransient->cbInstr;
9061	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9062	PVM pVM = pVCpu->CTX_SUFF(pVM);
9063	if (fIOString)
9064	{
9065	/*
9066	* INS/OUTS - I/O String instruction.
9067	*
9068	* Use instruction-information if available, otherwise fall back on
9069	* interpreting the instruction.
9070	*/
9071	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9072	#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9073	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9074	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.msr.vmx_basic_info))
9075	{
9076	rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9077	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9078	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9079	AssertRCReturn(rc2, rc2);
9080	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9081	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9082	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9083	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9084	if (fIOWrite)
9085	{
9086	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9087	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9088	//if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9089	// hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9090	// pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9091	}
9092	else
9093	{
9094	AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9095	("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9096	VERR_HMVMX_IPE_4);
9097	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9098	//if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9099	// hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9100	}
9101	}
9102	else
9103	{
9104	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9105	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9106	AssertRCReturn(rc2, rc2);
9107	rcStrict = IEMExecOne(pVCpu);
9108	}
9109	/** @todo IEM needs to be setting these flags somehow. */
9110	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9111	fUpdateRipAlready = true;
9112
9113	#else
9114	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9115	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9116	if (RT_SUCCESS(rcStrict))
9117	{
9118	if (fIOWrite)
9119	{
9120	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9121	(DISCPUMODE)pDis->uAddrMode, cbValue);
9122	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9123	}
9124	else
9125	{
9126	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9127	(DISCPUMODE)pDis->uAddrMode, cbValue);
9128	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9129	}
9130	}
9131	else
9132	{
9133	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9134	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9135	}
9136	#endif
9137	}
9138	else
9139	{
9140	/*
9141	* IN/OUT - I/O instruction.
9142	*/
9143	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9144	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9145	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9146	if (fIOWrite)
9147	{
9148	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9149	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9150	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9151	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9152	}
9153	else
9154	{
9155	uint32_t u32Result = 0;
9156	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9157	if (IOM_SUCCESS(rcStrict))
9158	{
9159	/* Save result of I/O IN instr. in AL/AX/EAX. */
9160	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9161	}
9162	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9163	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9164	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9165	}
9166	}
9167
9168	if (IOM_SUCCESS(rcStrict))
9169	{
9170	if (!fUpdateRipAlready)
9171	{
9172	pMixedCtx->rip += cbInstr;
9173	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9174	}
9175
9176	/*
9177	* If any I/O breakpoints are armed, we need to check if one triggered
9178	* and take appropriate action.
9179	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9180	*/
9181	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9182	AssertRCReturn(rc2, rc2);
9183
9184	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9185	* execution engines about whether hyper BPs and such are pending. */
9186	uint32_t const uDr7 = pMixedCtx->dr[7];
9187	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9188	&& X86_DR7_ANY_RW_IO(uDr7)
9189	&& (pMixedCtx->cr4 & X86_CR4_DE))
9190	\|\| DBGFBpIsHwIoArmed(pVM)))
9191	{
9192	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9193	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9194
9195	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9196	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9197	{
9198	/* Raise #DB. */
9199	if (fIsGuestDbgActive)
9200	ASMSetDR6(pMixedCtx->dr[6]);
9201	if (pMixedCtx->dr[7] != uDr7)
9202	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9203
9204	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9205	}
9206	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9207	else if ( rcStrict2 != VINF_SUCCESS
9208	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9209	rcStrict = rcStrict2;
9210	}
9211	}
9212
9213	#ifdef DEBUG
9214	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9215	Assert(!fIOWrite);
9216	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9217	Assert(fIOWrite);
9218	else
9219	{
9220	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9221	* statuses, that the VMM device and some others may return. See
9222	* IOM_SUCCESS() for guidance. */
9223	AssertMsg( RT_FAILURE(rcStrict)
9224	\|\| rcStrict == VINF_SUCCESS
9225	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9226	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9227	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9228	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9229	}
9230	#endif
9231
9232	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9233	return VBOXSTRICTRC_TODO(rcStrict);
9234	}
9235
9236
9237	/**
9238	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9239	* VM-exit.
9240	*/
9241	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9242	{
9243	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9244
9245	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9246	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9247	AssertRCReturn(rc, rc);
9248	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9249	{
9250	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9251	AssertRCReturn(rc, rc);
9252	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9253	{
9254	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9255
9256	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9257	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9258	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9259	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9260	{
9261	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9262	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9263
9264	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9265	Assert(!pVCpu->hm.s.Event.fPending);
9266	pVCpu->hm.s.Event.fPending = true;
9267	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9268	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9269	AssertRCReturn(rc, rc);
9270	if (fErrorCodeValid)
9271	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9272	else
9273	pVCpu->hm.s.Event.u32ErrCode = 0;
9274	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9275	&& uVector == X86_XCPT_PF)
9276	{
9277	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9278	}
9279
9280	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9281	}
9282	}
9283	}
9284
9285	/** @todo Emulate task switch someday, currently just going back to ring-3 for
9286	* emulation. */
9287	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9288	return VERR_EM_INTERPRETER;
9289	}
9290
9291
9292	/**
9293	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9294	*/
9295	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9296	{
9297	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9298	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9299	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9300	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9301	AssertRCReturn(rc, rc);
9302	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9303	return VINF_EM_DBG_STEPPED;
9304	}
9305
9306
9307	/**
9308	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9309	*/
9310	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9311	{
9312	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9313
9314	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9315	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9316	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9317	return VINF_SUCCESS;
9318	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9319	return rc;
9320
9321	#if 0
9322	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9323	* just sync the whole thing. */
9324	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9325	#else
9326	/* Aggressive state sync. for now. */
9327	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9328	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9329	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9330	#endif
9331	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9332	AssertRCReturn(rc, rc);
9333
9334	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9335	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9336	switch (uAccessType)
9337	{
9338	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9339	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9340	{
9341	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9342	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9343	{
9344	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9345	}
9346
9347	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9348	GCPhys &= PAGE_BASE_GC_MASK;
9349	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9350	PVM pVM = pVCpu->CTX_SUFF(pVM);
9351	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9352	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9353
9354	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9355	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9356	CPUMCTX2CORE(pMixedCtx), GCPhys);
9357	rc = VBOXSTRICTRC_VAL(rc2);
9358	Log4(("ApicAccess rc=%d\n", rc));
9359	if ( rc == VINF_SUCCESS
9360	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9361	\|\| rc == VERR_PAGE_NOT_PRESENT)
9362	{
9363	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9364	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9365	rc = VINF_SUCCESS;
9366	}
9367	break;
9368	}
9369
9370	default:
9371	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9372	rc = VINF_EM_RAW_EMULATE_INSTR;
9373	break;
9374	}
9375
9376	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9377	return rc;
9378	}
9379
9380
9381	/**
9382	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9383	* VM-exit.
9384	*/
9385	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9386	{
9387	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9388
9389	/* We should -not- get this VM-exit if the guest is debugging. */
9390	if (CPUMIsGuestDebugStateActive(pVCpu))
9391	{
9392	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9393	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9394	}
9395
9396	int rc = VERR_INTERNAL_ERROR_5;
9397	if ( !DBGFIsStepping(pVCpu)
9398	&& !pVCpu->hm.s.fSingleInstruction
9399	&& !CPUMIsHyperDebugStateActive(pVCpu))
9400	{
9401	/* Don't intercept MOV DRx and #DB any more. */
9402	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9403	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9404	AssertRCReturn(rc, rc);
9405
9406	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9407	{
9408	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9409	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9410	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9411	AssertRCReturn(rc, rc);
9412	#endif
9413	}
9414
9415	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9416	PVM pVM = pVCpu->CTX_SUFF(pVM);
9417	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9418	Assert(CPUMIsGuestDebugStateActive(pVCpu));
9419
9420	#ifdef VBOX_WITH_STATISTICS
9421	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9422	AssertRCReturn(rc, rc);
9423	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9424	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9425	else
9426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9427	#endif
9428	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9429	return VINF_SUCCESS;
9430	}
9431
9432	/*
9433	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9434	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9435	*/
9436	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9437	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9438	AssertRCReturn(rc, rc);
9439
9440	PVM pVM = pVCpu->CTX_SUFF(pVM);
9441	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9442	{
9443	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9444	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9445	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9446	if (RT_SUCCESS(rc))
9447	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9448	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9449	}
9450	else
9451	{
9452	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9453	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9454	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9456	}
9457
9458	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9459	if (RT_SUCCESS(rc))
9460	{
9461	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9462	AssertRCReturn(rc2, rc2);
9463	}
9464	return rc;
9465	}
9466
9467
9468	/**
9469	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9470	* Conditional VM-exit.
9471	*/
9472	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9473	{
9474	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9475	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9476
9477	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9478	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9479	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9480	return VINF_SUCCESS;
9481	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9482	return rc;
9483
9484	RTGCPHYS GCPhys = 0;
9485	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9486
9487	#if 0
9488	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9489	#else
9490	/* Aggressive state sync. for now. */
9491	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9492	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9493	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9494	#endif
9495	AssertRCReturn(rc, rc);
9496
9497	/*
9498	* If we succeed, resume guest execution.
9499	* If we fail in interpreting the instruction because we couldn't get the guest physical address
9500	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
9501	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
9502	* weird case. See @bugref{6043}.
9503	*/
9504	PVM pVM = pVCpu->CTX_SUFF(pVM);
9505	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
9506	rc = VBOXSTRICTRC_VAL(rc2);
9507	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
9508	if ( rc == VINF_SUCCESS
9509	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9510	\|\| rc == VERR_PAGE_NOT_PRESENT)
9511	{
9512	/* Successfully handled MMIO operation. */
9513	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9514	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9515	rc = VINF_SUCCESS;
9516	}
9517	return rc;
9518	}
9519
9520
9521	/**
9522	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
9523	* VM-exit.
9524	*/
9525	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9526	{
9527	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9528	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9529
9530	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9531	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9532	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9533	return VINF_SUCCESS;
9534	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9535	return rc;
9536
9537	RTGCPHYS GCPhys = 0;
9538	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9539	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9540	#if 0
9541	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9542	#else
9543	/* Aggressive state sync. for now. */
9544	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9545	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9546	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9547	#endif
9548	AssertRCReturn(rc, rc);
9549
9550	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
9551	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
9552
9553	RTGCUINT uErrorCode = 0;
9554	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
9555	uErrorCode \|= X86_TRAP_PF_ID;
9556	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
9557	uErrorCode \|= X86_TRAP_PF_RW;
9558	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
9559	uErrorCode \|= X86_TRAP_PF_P;
9560
9561	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
9562
9563	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
9564	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9565
9566	/* Handle the pagefault trap for the nested shadow table. */
9567	PVM pVM = pVCpu->CTX_SUFF(pVM);
9568	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
9569	TRPMResetTrap(pVCpu);
9570
9571	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
9572	if ( rc == VINF_SUCCESS
9573	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9574	\|\| rc == VERR_PAGE_NOT_PRESENT)
9575	{
9576	/* Successfully synced our nested page tables. */
9577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
9578	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
9579	return VINF_SUCCESS;
9580	}
9581
9582	Log4(("EPT return to ring-3 rc=%d\n"));
9583	return rc;
9584	}
9585
9586	/** @} */
9587
9588	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9589	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
9590	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9591
9592	/** @name VM-exit exception handlers.
9593	* @{
9594	*/
9595
9596	/**
9597	* VM-exit exception handler for #MF (Math Fault: floating point exception).
9598	*/
9599	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9600	{
9601	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9602	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
9603
9604	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9605	AssertRCReturn(rc, rc);
9606
9607	if (!(pMixedCtx->cr0 & X86_CR0_NE))
9608	{
9609	/* Old-style FPU error reporting needs some extra work. */
9610	/** @todo don't fall back to the recompiler, but do it manually. */
9611	return VERR_EM_INTERPRETER;
9612	}
9613
9614	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9615	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9616	return rc;
9617	}
9618
9619
9620	/**
9621	* VM-exit exception handler for #BP (Breakpoint exception).
9622	*/
9623	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9624	{
9625	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9626	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
9627
9628	/** @todo Try optimize this by not saving the entire guest state unless
9629	* really needed. */
9630	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9631	AssertRCReturn(rc, rc);
9632
9633	PVM pVM = pVCpu->CTX_SUFF(pVM);
9634	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9635	if (rc == VINF_EM_RAW_GUEST_TRAP)
9636	{
9637	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9638	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9639	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9640	AssertRCReturn(rc, rc);
9641
9642	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9643	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9644	}
9645
9646	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
9647	return rc;
9648	}
9649
9650
9651	/**
9652	* VM-exit exception handler for #DB (Debug exception).
9653	*/
9654	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9655	{
9656	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9657	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
9658	Log6(("XcptDB\n"));
9659
9660	/*
9661	* Get the DR6-like values from the exit qualification and pass it to DBGF
9662	* for processing.
9663	*/
9664	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9665	AssertRCReturn(rc, rc);
9666
9667	/* If we sat the trap flag above, we have to clear it. */
9668	if (pVCpu->hm.s.fClearTrapFlag)
9669	{
9670	pVCpu->hm.s.fClearTrapFlag = false;
9671	pMixedCtx->eflags.Bits.u1TF = 0;
9672	}
9673
9674	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
9675	uint64_t uDR6 = X86_DR6_INIT_VAL;
9676	uDR6 \|= ( pVmxTransient->uExitQualification
9677	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
9678
9679	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
9680	if (rc == VINF_EM_RAW_GUEST_TRAP)
9681	{
9682	/*
9683	* The exception was for the guest. Update DR6, DR7.GD and
9684	* IA32_DEBUGCTL.LBR before forwarding it.
9685	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
9686	*/
9687	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
9688	pMixedCtx->dr[6] \|= uDR6;
9689	if (CPUMIsGuestDebugStateActive(pVCpu))
9690	ASMSetDR6(pMixedCtx->dr[6]);
9691
9692	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9693	AssertRCReturn(rc, rc);
9694
9695	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
9696	pMixedCtx->dr[7] &= ~X86_DR7_GD;
9697
9698	/* Paranoia. */
9699	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
9700	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
9701
9702	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
9703	AssertRCReturn(rc, rc);
9704
9705	/*
9706	* Raise #DB in the guest.
9707	*/
9708	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9709	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9710	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9711	AssertRCReturn(rc, rc);
9712	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9713	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9714	return VINF_SUCCESS;
9715	}
9716
9717	/*
9718	* Not a guest trap, must be a hypervisor related debug event then.
9719	* Update DR6 in case someone is interested in it.
9720	*/
9721	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
9722	AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
9723	CPUMSetHyperDR6(pVCpu, uDR6);
9724
9725	return rc;
9726	}
9727
9728
9729	/**
9730	* VM-exit exception handler for #NM (Device-not-available exception: floating
9731	* point exception).
9732	*/
9733	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9734	{
9735	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9736
9737	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9738	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
9739	#endif
9740
9741	/* We require CR0 and EFER. EFER is always up-to-date. */
9742	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9743	AssertRCReturn(rc, rc);
9744
9745	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
9746	PVM pVM = pVCpu->CTX_SUFF(pVM);
9747	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
9748	if (rc == VINF_SUCCESS)
9749	{
9750	Assert(CPUMIsGuestFPUStateActive(pVCpu));
9751	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9752	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
9753	return VINF_SUCCESS;
9754	}
9755
9756	/* Forward #NM to the guest. */
9757	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
9758	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9759	AssertRCReturn(rc, rc);
9760	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9761	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
9762	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
9763	return rc;
9764	}
9765
9766
9767	/**
9768	* VM-exit exception handler for #GP (General-protection exception).
9769	*
9770	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
9771	*/
9772	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9773	{
9774	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9775	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
9776
9777	int rc = VERR_INTERNAL_ERROR_5;
9778	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9779	{
9780	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9781	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
9782	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9783	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9784	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9785	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9786	AssertRCReturn(rc, rc);
9787	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
9788	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
9789	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9790	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9791	return rc;
9792	#else
9793	/* We don't intercept #GP. */
9794	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
9795	return VERR_VMX_UNEXPECTED_EXCEPTION;
9796	#endif
9797	}
9798
9799	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9800	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
9801
9802	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
9803	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9804	AssertRCReturn(rc, rc);
9805
9806	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9807	uint32_t cbOp = 0;
9808	PVM pVM = pVCpu->CTX_SUFF(pVM);
9809	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
9810	if (RT_SUCCESS(rc))
9811	{
9812	rc = VINF_SUCCESS;
9813	Assert(cbOp == pDis->cbInstr);
9814	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9815	switch (pDis->pCurInstr->uOpcode)
9816	{
9817	case OP_CLI:
9818	{
9819	pMixedCtx->eflags.Bits.u1IF = 0;
9820	pMixedCtx->rip += pDis->cbInstr;
9821	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9822	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
9823	break;
9824	}
9825
9826	case OP_STI:
9827	{
9828	pMixedCtx->eflags.Bits.u1IF = 1;
9829	pMixedCtx->rip += pDis->cbInstr;
9830	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
9831	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9832	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9833	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
9834	break;
9835	}
9836
9837	case OP_HLT:
9838	{
9839	rc = VINF_EM_HALT;
9840	pMixedCtx->rip += pDis->cbInstr;
9841	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9843	break;
9844	}
9845
9846	case OP_POPF:
9847	{
9848	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9849	uint32_t cbParm = 0;
9850	uint32_t uMask = 0;
9851	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9852	{
9853	cbParm = 4;
9854	uMask = 0xffffffff;
9855	}
9856	else
9857	{
9858	cbParm = 2;
9859	uMask = 0xffff;
9860	}
9861
9862	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
9863	RTGCPTR GCPtrStack = 0;
9864	X86EFLAGS uEflags;
9865	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9866	&GCPtrStack);
9867	if (RT_SUCCESS(rc))
9868	{
9869	Assert(sizeof(uEflags.u32) >= cbParm);
9870	uEflags.u32 = 0;
9871	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
9872	}
9873	if (RT_FAILURE(rc))
9874	{
9875	rc = VERR_EM_INTERPRETER;
9876	break;
9877	}
9878	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
9879	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9880	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
9881	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
9882	pMixedCtx->eflags.Bits.u1RF = 0;
9883	pMixedCtx->esp += cbParm;
9884	pMixedCtx->esp &= uMask;
9885	pMixedCtx->rip += pDis->cbInstr;
9886	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
9887	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
9888	break;
9889	}
9890
9891	case OP_PUSHF:
9892	{
9893	uint32_t cbParm = 0;
9894	uint32_t uMask = 0;
9895	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9896	{
9897	cbParm = 4;
9898	uMask = 0xffffffff;
9899	}
9900	else
9901	{
9902	cbParm = 2;
9903	uMask = 0xffff;
9904	}
9905
9906	/* Get the stack pointer & push the contents of eflags onto the stack. */
9907	RTGCPTR GCPtrStack = 0;
9908	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
9909	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
9910	if (RT_FAILURE(rc))
9911	{
9912	rc = VERR_EM_INTERPRETER;
9913	break;
9914	}
9915	X86EFLAGS uEflags;
9916	uEflags = pMixedCtx->eflags;
9917	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
9918	uEflags.Bits.u1RF = 0;
9919	uEflags.Bits.u1VM = 0;
9920
9921	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
9922	if (RT_FAILURE(rc))
9923	{
9924	rc = VERR_EM_INTERPRETER;
9925	break;
9926	}
9927	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
9928	pMixedCtx->esp -= cbParm;
9929	pMixedCtx->esp &= uMask;
9930	pMixedCtx->rip += pDis->cbInstr;
9931	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
9932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
9933	break;
9934	}
9935
9936	case OP_IRET:
9937	{
9938	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
9939	* instruction reference. */
9940	RTGCPTR GCPtrStack = 0;
9941	uint32_t uMask = 0xffff;
9942	uint16_t aIretFrame[3];
9943	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
9944	{
9945	rc = VERR_EM_INTERPRETER;
9946	break;
9947	}
9948	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9949	&GCPtrStack);
9950	if (RT_SUCCESS(rc))
9951	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
9952	if (RT_FAILURE(rc))
9953	{
9954	rc = VERR_EM_INTERPRETER;
9955	break;
9956	}
9957	pMixedCtx->eip = 0;
9958	pMixedCtx->ip = aIretFrame[0];
9959	pMixedCtx->cs.Sel = aIretFrame[1];
9960	pMixedCtx->cs.ValidSel = aIretFrame[1];
9961	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
9962	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9963	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
9964	pMixedCtx->sp += sizeof(aIretFrame);
9965	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
9966	\| HM_CHANGED_GUEST_RFLAGS;
9967	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9968	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9969	break;
9970	}
9971
9972	case OP_INT:
9973	{
9974	uint16_t uVector = pDis->Param1.uValue & 0xff;
9975	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9976	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9977	break;
9978	}
9979
9980	case OP_INTO:
9981	{
9982	if (pMixedCtx->eflags.Bits.u1OF)
9983	{
9984	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9985	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9986	}
9987	break;
9988	}
9989
9990	default:
9991	{
9992	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9993	EMCODETYPE_SUPERVISOR);
9994	rc = VBOXSTRICTRC_VAL(rc2);
9995	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9996	Log4(("#GP rc=%Rrc\n", rc));
9997	break;
9998	}
9999	}
10000	}
10001	else
10002	rc = VERR_EM_INTERPRETER;
10003
10004	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10005	("#GP Unexpected rc=%Rrc\n", rc));
10006	return rc;
10007	}
10008
10009
10010	/**
10011	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10012	* the exception reported in the VMX transient structure back into the VM.
10013	*
10014	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
10015	* up-to-date.
10016	*/
10017	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10018	{
10019	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10020
10021	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10022	hmR0VmxCheckExitDueToEventDelivery(). */
10023	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10024	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10025	AssertRCReturn(rc, rc);
10026	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10027
10028	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10029	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10030	return VINF_SUCCESS;
10031	}
10032
10033
10034	/**
10035	* VM-exit exception handler for #PF (Page-fault exception).
10036	*/
10037	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10038	{
10039	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10040	PVM pVM = pVCpu->CTX_SUFF(pVM);
10041	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10042	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10043	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10044	AssertRCReturn(rc, rc);
10045
10046	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10047	if (pVM->hm.s.fNestedPaging)
10048	{
10049	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10050	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10051	{
10052	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10053	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10054	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10055	}
10056	else
10057	{
10058	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10059	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10060	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10061	}
10062	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10063	return rc;
10064	}
10065	#else
10066	Assert(!pVM->hm.s.fNestedPaging);
10067	#endif
10068
10069	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10070	AssertRCReturn(rc, rc);
10071
10072	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10073	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10074
10075	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10076	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10077	(RTGCPTR)pVmxTransient->uExitQualification);
10078
10079	Log4(("#PF: rc=%Rrc\n", rc));
10080	if (rc == VINF_SUCCESS)
10081	{
10082	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10083	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10084	* memory? We don't update the whole state here... */
10085	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
10086	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
10087	TRPMResetTrap(pVCpu);
10088	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10089	return rc;
10090	}
10091	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10092	{
10093	if (!pVmxTransient->fVectoringPF)
10094	{
10095	/* It's a guest page fault and needs to be reflected to the guest. */
10096	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10097	TRPMResetTrap(pVCpu);
10098	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10099	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10100	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10101	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10102	}
10103	else
10104	{
10105	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10106	TRPMResetTrap(pVCpu);
10107	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10108	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10109	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10110	}
10111
10112	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10113	return VINF_SUCCESS;
10114	}
10115
10116	TRPMResetTrap(pVCpu);
10117	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10118	return rc;
10119	}
10120
10121	/** @} */
10122

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 47719

Download in other formats: