HMVMXR0.cpp@ 48360

Last change on this file since 48360 was 48360, checked in by vboxsync, 11 years ago
HMVMXR0.cpp: Made hmR0VmxSetupVMRunHandler trigger the necessary host (and guest) field updates, and moved it up so the HM_CHANGED_VMX_EXIT_CTLS/HM_CHANGED_VMX_ENTRY_CTLS would take affect before executing guest code again.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 446.3 KB

Line
1	/* $Id: HMVMXR0.cpp 48360 2013-09-06 14:29:56Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#endif
45
46
47	/*******************************************************************************
48	* Defined Constants And Macros *
49	*******************************************************************************/
50	#if defined(RT_ARCH_AMD64)
51	# define HMVMX_IS_64BIT_HOST_MODE() (true)
52	typedef RTHCUINTREG HMVMXHCUINTREG;
53	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54	extern "C" uint32_t g_fVMXIs64bitHost;
55	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56	typedef uint64_t HMVMXHCUINTREG;
57	#else
58	# define HMVMX_IS_64BIT_HOST_MODE() (false)
59	typedef RTHCUINTREG HMVMXHCUINTREG;
60	#endif
61
62	/** Use the function table. */
63	#define HMVMX_USE_FUNCTION_TABLE
64
65	/** Determine which tagged-TLB flush handler to use. */
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71	/** @name Updated-guest-state flags.
72	* @{ */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113	/** @} */
114
115	/** @name
116	* Flags to skip redundant reads of some common VMCS fields that are not part of
117	* the guest-CPU state but are in the transient structure.
118	*/
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125	/** @} */
126
127	/** @name
128	* States of the VMCS.
129	*
130	* This does not reflect all possible VMCS states but currently only those
131	* needed for maintaining the VMCS consistently even when thread-context hooks
132	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
133	*/
134	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137	/** @} */
138
139	/**
140	* Exception bitmap mask for real-mode guests (real-on-v86).
141	*
142	* We need to intercept all exceptions manually (except #PF). #NM is also
143	* handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
144	* intercepted even in real-mode if we have Nested Paging support.
145	*/
146	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
147	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
148	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
149	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
150	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
152	\| RT_BIT(X86_XCPT_XF))
153
154	/**
155	* Exception bitmap mask for all contributory exceptions.
156	*
157	* Page fault is deliberately excluded here as it's conditional as to whether
158	* it's contributory or benign. Page faults are handled separately.
159	*/
160	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
161	\| RT_BIT(X86_XCPT_DE))
162
163	/** Maximum VM-instruction error number. */
164	#define HMVMX_INSTR_ERROR_MAX 28
165
166	/** Profiling macro. */
167	#ifdef HM_PROFILE_EXIT_DISPATCH
168	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170	#else
171	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173	#endif
174
175	/** Assert that preemption is disabled or covered by thread-context hooks. */
176	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
177	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179	/** Assert that we haven't migrated CPUs when thread-context hooks are not
180	* used. */
181	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183	("Illegal migration! Entered on CPU %u Current %u\n", \
184	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186	/*******************************************************************************
187	* Structures and Typedefs *
188	*******************************************************************************/
189	/**
190	* VMX transient state.
191	*
192	* A state structure for holding miscellaneous information across
193	* VMX non-root operation and restored after the transition.
194	*/
195	typedef struct VMXTRANSIENT
196	{
197	/** The host's rflags/eflags. */
198	RTCCUINTREG uEflags;
199	#if HC_ARCH_BITS == 32
200	uint32_t u32Alignment0;
201	#endif
202	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
203	uint64_t u64LStarMsr;
204	/** The guest's TPR value used for TPR shadowing. */
205	uint8_t u8GuestTpr;
206	/** Alignment. */
207	uint8_t abAlignment0[7];
208
209	/** The basic VM-exit reason. */
210	uint16_t uExitReason;
211	/** Alignment. */
212	uint16_t u16Alignment0;
213	/** The VM-exit interruption error code. */
214	uint32_t uExitIntrErrorCode;
215	/** The VM-exit exit qualification. */
216	uint64_t uExitQualification;
217
218	/** The VM-exit interruption-information field. */
219	uint32_t uExitIntrInfo;
220	/** The VM-exit instruction-length field. */
221	uint32_t cbInstr;
222	/** The VM-exit instruction-information field. */
223	union
224	{
225	/** Plain unsigned int representation. */
226	uint32_t u;
227	/** INS and OUTS information. */
228	struct
229	{
230	uint32_t u6Reserved0 : 6;
231	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
232	uint32_t u3AddrSize : 3;
233	uint32_t u5Reserved1 : 5;
234	/** The segment register (X86_SREG_XXX). */
235	uint32_t iSegReg : 3;
236	uint32_t uReserved2 : 14;
237	} StrIo;
238	} ExitInstrInfo;
239	/** Whether the VM-entry failed or not. */
240	bool fVMEntryFailed;
241	/** Alignment. */
242	uint8_t abAlignment1[3];
243
244	/** The VM-entry interruption-information field. */
245	uint32_t uEntryIntrInfo;
246	/** The VM-entry exception error code field. */
247	uint32_t uEntryXcptErrorCode;
248	/** The VM-entry instruction length field. */
249	uint32_t cbEntryInstr;
250
251	/** IDT-vectoring information field. */
252	uint32_t uIdtVectoringInfo;
253	/** IDT-vectoring error code. */
254	uint32_t uIdtVectoringErrorCode;
255
256	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
257	uint32_t fVmcsFieldsRead;
258	/** Whether TSC-offsetting should be setup before VM-entry. */
259	bool fUpdateTscOffsettingAndPreemptTimer;
260	/** Whether the VM-exit was caused by a page-fault during delivery of a
261	* contributory exception or a page-fault. */
262	bool fVectoringPF;
263	} VMXTRANSIENT;
264	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
265	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
266	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
267	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
268	/** Pointer to VMX transient state. */
269	typedef VMXTRANSIENT *PVMXTRANSIENT;
270
271
272	/**
273	* MSR-bitmap read permissions.
274	*/
275	typedef enum VMXMSREXITREAD
276	{
277	/** Reading this MSR causes a VM-exit. */
278	VMXMSREXIT_INTERCEPT_READ = 0xb,
279	/** Reading this MSR does not cause a VM-exit. */
280	VMXMSREXIT_PASSTHRU_READ
281	} VMXMSREXITREAD;
282
283	/**
284	* MSR-bitmap write permissions.
285	*/
286	typedef enum VMXMSREXITWRITE
287	{
288	/** Writing to this MSR causes a VM-exit. */
289	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
290	/** Writing to this MSR does not cause a VM-exit. */
291	VMXMSREXIT_PASSTHRU_WRITE
292	} VMXMSREXITWRITE;
293
294	/**
295	* VMX VM-exit handler.
296	*
297	* @returns VBox status code.
298	* @param pVCpu Pointer to the VMCPU.
299	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
300	* out-of-sync. Make sure to update the required
301	* fields before using them.
302	* @param pVmxTransient Pointer to the VMX-transient structure.
303	*/
304	#ifndef HMVMX_USE_FUNCTION_TABLE
305	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	#else
307	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	/** Pointer to VM-exit handler. */
309	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
310	#endif
311
312
313	/*******************************************************************************
314	* Internal Functions *
315	*******************************************************************************/
316	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
317	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
318	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
319	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
320	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
321	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
322	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
323	#endif
324	#ifndef HMVMX_USE_FUNCTION_TABLE
325	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
326	# define HMVMX_EXIT_DECL static int
327	#else
328	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
329	#endif
330
331	/** @name VM-exit handlers.
332	* @{
333	*/
334	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
335	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
336	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
337	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
338	static FNVMXEXITHANDLER hmR0VmxExitSipi;
339	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
340	static FNVMXEXITHANDLER hmR0VmxExitSmi;
341	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
342	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
343	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
344	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
345	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
346	static FNVMXEXITHANDLER hmR0VmxExitHlt;
347	static FNVMXEXITHANDLER hmR0VmxExitInvd;
348	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
349	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
350	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
351	static FNVMXEXITHANDLER hmR0VmxExitRsm;
352	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
353	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
354	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
355	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
356	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
357	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
358	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
359	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
360	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
361	static FNVMXEXITHANDLER hmR0VmxExitMwait;
362	static FNVMXEXITHANDLER hmR0VmxExitMtf;
363	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
364	static FNVMXEXITHANDLER hmR0VmxExitPause;
365	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
366	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
367	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
368	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
369	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
370	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
371	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
372	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
373	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
374	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
375	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
376	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
377	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
378	/** @} */
379
380	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
381	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
382	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
383	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
385	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
387	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
388
389	/*******************************************************************************
390	* Global Variables *
391	*******************************************************************************/
392	#ifdef HMVMX_USE_FUNCTION_TABLE
393
394	/**
395	* VMX_EXIT dispatch table.
396	*/
397	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
398	{
399	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
400	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
401	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
402	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
403	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
404	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
405	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
406	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
407	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
408	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
409	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
410	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
411	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
412	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
413	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
414	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
415	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
416	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
417	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
418	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
419	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
420	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
421	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
422	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
423	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
424	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
425	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
426	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
427	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
428	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
429	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
430	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
431	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
432	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
433	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
434	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
435	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
436	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
437	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
438	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
439	/* 40 UNDEFINED */ hmR0VmxExitPause,
440	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
441	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
442	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
443	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
444	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
445	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
446	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
447	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
448	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
449	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
450	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
451	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
452	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
453	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
454	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
455	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
456	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
457	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
458	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
459	};
460	#endif /* HMVMX_USE_FUNCTION_TABLE */
461
462	#ifdef VBOX_STRICT
463	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
464	{
465	/* 0 */ "(Not Used)",
466	/* 1 */ "VMCALL executed in VMX root operation.",
467	/* 2 */ "VMCLEAR with invalid physical address.",
468	/* 3 */ "VMCLEAR with VMXON pointer.",
469	/* 4 */ "VMLAUNCH with non-clear VMCS.",
470	/* 5 */ "VMRESUME with non-launched VMCS.",
471	/* 6 */ "VMRESUME after VMXOFF",
472	/* 7 */ "VM entry with invalid control fields.",
473	/* 8 */ "VM entry with invalid host state fields.",
474	/* 9 */ "VMPTRLD with invalid physical address.",
475	/* 10 */ "VMPTRLD with VMXON pointer.",
476	/* 11 */ "VMPTRLD with incorrect revision identifier.",
477	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
478	/* 13 */ "VMWRITE to read-only VMCS component.",
479	/* 14 */ "(Not Used)",
480	/* 15 */ "VMXON executed in VMX root operation.",
481	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
482	/* 17 */ "VM entry with non-launched executing VMCS.",
483	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
484	/* 19 */ "VMCALL with non-clear VMCS.",
485	/* 20 */ "VMCALL with invalid VM-exit control fields.",
486	/* 21 */ "(Not Used)",
487	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
488	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
489	/* 24 */ "VMCALL with invalid SMM-monitor features.",
490	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
491	/* 26 */ "VM entry with events blocked by MOV SS.",
492	/* 27 */ "(Not Used)",
493	/* 28 */ "Invalid operand to INVEPT/INVVPID."
494	};
495	#endif /* VBOX_STRICT */
496
497
498
499	/**
500	* Updates the VM's last error record. If there was a VMX instruction error,
501	* reads the error data from the VMCS and updates VCPU's last error record as
502	* well.
503	*
504	* @param pVM Pointer to the VM.
505	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
506	* VERR_VMX_UNABLE_TO_START_VM or
507	* VERR_VMX_INVALID_VMCS_FIELD).
508	* @param rc The error code.
509	*/
510	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
511	{
512	AssertPtr(pVM);
513	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
514	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
515	{
516	AssertPtrReturnVoid(pVCpu);
517	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
518	}
519	pVM->hm.s.lLastError = rc;
520	}
521
522
523	/**
524	* Reads the VM-entry interruption-information field from the VMCS into the VMX
525	* transient structure.
526	*
527	* @returns VBox status code.
528	* @param pVmxTransient Pointer to the VMX transient structure.
529	*
530	* @remarks No-long-jump zone!!!
531	*/
532	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
533	{
534	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
535	AssertRCReturn(rc, rc);
536	return VINF_SUCCESS;
537	}
538
539
540	/**
541	* Reads the VM-entry exception error code field from the VMCS into
542	* the VMX transient structure.
543	*
544	* @returns VBox status code.
545	* @param pVmxTransient Pointer to the VMX transient structure.
546	*
547	* @remarks No-long-jump zone!!!
548	*/
549	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
550	{
551	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
552	AssertRCReturn(rc, rc);
553	return VINF_SUCCESS;
554	}
555
556
557	/**
558	* Reads the VM-entry exception error code field from the VMCS into
559	* the VMX transient structure.
560	*
561	* @returns VBox status code.
562	* @param pVCpu Pointer to the VMCPU.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*
565	* @remarks No-long-jump zone!!!
566	*/
567	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
570	AssertRCReturn(rc, rc);
571	return VINF_SUCCESS;
572	}
573
574
575	/**
576	* Reads the VM-exit interruption-information field from the VMCS into the VMX
577	* transient structure.
578	*
579	* @returns VBox status code.
580	* @param pVCpu Pointer to the VMCPU.
581	* @param pVmxTransient Pointer to the VMX transient structure.
582	*/
583	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
584	{
585	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
586	{
587	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
588	AssertRCReturn(rc, rc);
589	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
590	}
591	return VINF_SUCCESS;
592	}
593
594
595	/**
596	* Reads the VM-exit interruption error code from the VMCS into the VMX
597	* transient structure.
598	*
599	* @returns VBox status code.
600	* @param pVCpu Pointer to the VMCPU.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit instruction length field from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVCpu Pointer to the VMCPU.
621	* @param pVmxTransient Pointer to the VMX transient structure.
622	*/
623	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
624	{
625	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
626	{
627	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
628	AssertRCReturn(rc, rc);
629	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
630	}
631	return VINF_SUCCESS;
632	}
633
634
635	/**
636	* Reads the VM-exit instruction-information field from the VMCS into
637	* the VMX transient structure.
638	*
639	* @returns VBox status code.
640	* @param pVCpu The cross context per CPU structure.
641	* @param pVmxTransient Pointer to the VMX transient structure.
642	*/
643	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
644	{
645	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
646	{
647	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
648	AssertRCReturn(rc, rc);
649	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
650	}
651	return VINF_SUCCESS;
652	}
653
654
655	/**
656	* Reads the exit qualification from the VMCS into the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVCpu Pointer to the VMCPU.
660	* @param pVmxTransient Pointer to the VMX transient structure.
661	*/
662	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
663	{
664	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
665	{
666	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
667	AssertRCReturn(rc, rc);
668	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
669	}
670	return VINF_SUCCESS;
671	}
672
673
674	/**
675	* Reads the IDT-vectoring information field from the VMCS into the VMX
676	* transient structure.
677	*
678	* @returns VBox status code.
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*
681	* @remarks No-long-jump zone!!!
682	*/
683	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
684	{
685	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
686	{
687	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
688	AssertRCReturn(rc, rc);
689	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
690	}
691	return VINF_SUCCESS;
692	}
693
694
695	/**
696	* Reads the IDT-vectoring error code from the VMCS into the VMX
697	* transient structure.
698	*
699	* @returns VBox status code.
700	* @param pVmxTransient Pointer to the VMX transient structure.
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Enters VMX root mode operation on the current CPU.
716	*
717	* @returns VBox status code.
718	* @param pVM Pointer to the VM (optional, can be NULL, after
719	* a resume).
720	* @param HCPhysCpuPage Physical address of the VMXON region.
721	* @param pvCpuPage Pointer to the VMXON region.
722	*/
723	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
724	{
725	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
726	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
727	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
728
729	if (pVM)
730	{
731	/* Write the VMCS revision dword to the VMXON region. */
732	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
733	}
734
735	/* Enable the VMX bit in CR4 if necessary. */
736	RTCCUINTREG uCr4 = ASMGetCR4();
737	if (!(uCr4 & X86_CR4_VMXE))
738	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
739
740	/* Enter VMX root mode. */
741	int rc = VMXEnable(HCPhysCpuPage);
742	if (RT_FAILURE(rc))
743	ASMSetCR4(uCr4);
744
745	return rc;
746	}
747
748
749	/**
750	* Exits VMX root mode operation on the current CPU.
751	*
752	* @returns VBox status code.
753	*/
754	static int hmR0VmxLeaveRootMode(void)
755	{
756	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758	/* If we're for some reason not in VMX root mode, then don't leave it. */
759	RTCCUINTREG uHostCR4 = ASMGetCR4();
760	if (uHostCR4 & X86_CR4_VMXE)
761	{
762	/* Exit VMX root mode and clear the VMX bit in CR4. */
763	VMXDisable();
764	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
765	return VINF_SUCCESS;
766	}
767
768	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
769	}
770
771
772	/**
773	* Allocates and maps one physically contiguous page. The allocated page is
774	* zero'd out. (Used by various VT-x structures).
775	*
776	* @returns IPRT status code.
777	* @param pMemObj Pointer to the ring-0 memory object.
778	* @param ppVirt Where to store the virtual address of the
779	* allocation.
780	* @param pPhys Where to store the physical address of the
781	* allocation.
782	*/
783	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
784	{
785	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
786	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
787	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
788
789	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
790	if (RT_FAILURE(rc))
791	return rc;
792	ppVirt = RTR0MemObjAddress(pMemObj);
793	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
794	ASMMemZero32(*ppVirt, PAGE_SIZE);
795	return VINF_SUCCESS;
796	}
797
798
799	/**
800	* Frees and unmaps an allocated physical page.
801	*
802	* @param pMemObj Pointer to the ring-0 memory object.
803	* @param ppVirt Where to re-initialize the virtual address of
804	* allocation as 0.
805	* @param pHCPhys Where to re-initialize the physical address of the
806	* allocation as 0.
807	*/
808	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
809	{
810	AssertPtr(pMemObj);
811	AssertPtr(ppVirt);
812	AssertPtr(pHCPhys);
813	if (*pMemObj != NIL_RTR0MEMOBJ)
814	{
815	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
816	AssertRC(rc);
817	*pMemObj = NIL_RTR0MEMOBJ;
818	*ppVirt = 0;
819	*pHCPhys = 0;
820	}
821	}
822
823
824	/**
825	* Worker function to free VT-x related structures.
826	*
827	* @returns IPRT status code.
828	* @param pVM Pointer to the VM.
829	*/
830	static void hmR0VmxStructsFree(PVM pVM)
831	{
832	for (VMCPUID i = 0; i < pVM->cCpus; i++)
833	{
834	PVMCPU pVCpu = &pVM->aCpus[i];
835	AssertPtr(pVCpu);
836
837	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
838	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
839	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
840	#endif
841
842	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
843	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
844
845	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
846	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
847	}
848
849	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
850	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
851	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
852	#endif
853	}
854
855
856	/**
857	* Worker function to allocate VT-x related VM structures.
858	*
859	* @returns IPRT status code.
860	* @param pVM Pointer to the VM.
861	*/
862	static int hmR0VmxStructsAlloc(PVM pVM)
863	{
864	/*
865	* Initialize members up-front so we can cleanup properly on allocation failure.
866	*/
867	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
868	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
869	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
870	pVM->hm.s.vmx.HCPhys##a_Name = 0;
871
872	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
873	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
874	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
875	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
876
877	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
878	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
879	#endif
880	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
881
882	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
883	for (VMCPUID i = 0; i < pVM->cCpus; i++)
884	{
885	PVMCPU pVCpu = &pVM->aCpus[i];
886	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
887	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
888	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
889	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
890	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
891	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
892	#endif
893	}
894	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
895	#undef VMXLOCAL_INIT_VM_MEMOBJ
896
897	/*
898	* Allocate all the VT-x structures.
899	*/
900	int rc = VINF_SUCCESS;
901	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
902	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
903	if (RT_FAILURE(rc))
904	goto cleanup;
905	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
906	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
907	#endif
908
909	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
910	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
911	{
912	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
913	&pVM->hm.s.vmx.HCPhysApicAccess);
914	if (RT_FAILURE(rc))
915	goto cleanup;
916	}
917
918	/*
919	* Initialize per-VCPU VT-x structures.
920	*/
921	for (VMCPUID i = 0; i < pVM->cCpus; i++)
922	{
923	PVMCPU pVCpu = &pVM->aCpus[i];
924	AssertPtr(pVCpu);
925
926	/* Allocate the VM control structure (VMCS). */
927	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
928	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
929	if (RT_FAILURE(rc))
930	goto cleanup;
931
932	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
933	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
934	{
935	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
936	&pVCpu->hm.s.vmx.HCPhysVirtApic);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	}
940
941	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
942	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943	{
944	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
945	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
946	if (RT_FAILURE(rc))
947	goto cleanup;
948	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
949	}
950
951	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
952	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
953	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
954	if (RT_FAILURE(rc))
955	goto cleanup;
956
957	/* Allocate the VM-exit MSR-load page for the host MSRs. */
958	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
959	if (RT_FAILURE(rc))
960	goto cleanup;
961	#endif
962	}
963
964	return VINF_SUCCESS;
965
966	cleanup:
967	hmR0VmxStructsFree(pVM);
968	return rc;
969	}
970
971
972	/**
973	* Does global VT-x initialization (called during module initialization).
974	*
975	* @returns VBox status code.
976	*/
977	VMMR0DECL(int) VMXR0GlobalInit(void)
978	{
979	#ifdef HMVMX_USE_FUNCTION_TABLE
980	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
981	# ifdef VBOX_STRICT
982	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
983	Assert(g_apfnVMExitHandlers[i]);
984	# endif
985	#endif
986	return VINF_SUCCESS;
987	}
988
989
990	/**
991	* Does global VT-x termination (called during module termination).
992	*/
993	VMMR0DECL(void) VMXR0GlobalTerm()
994	{
995	/* Nothing to do currently. */
996	}
997
998
999	/**
1000	* Sets up and activates VT-x on the current CPU.
1001	*
1002	* @returns VBox status code.
1003	* @param pCpu Pointer to the global CPU info struct.
1004	* @param pVM Pointer to the VM (can be NULL after a host resume
1005	* operation).
1006	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1007	* fEnabledByHost is true).
1008	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1009	* @a fEnabledByHost is true).
1010	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1011	* enable VT-x on the host.
1012	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1013	*/
1014	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1015	void *pvMsrs)
1016	{
1017	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1018	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1019	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1020
1021	/* Enable VT-x if it's not already enabled by the host. */
1022	if (!fEnabledByHost)
1023	{
1024	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1025	if (RT_FAILURE(rc))
1026	return rc;
1027	}
1028
1029	/*
1030	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1031	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1032	*/
1033	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1034	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1035	{
1036	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1037	pCpu->fFlushAsidBeforeUse = false;
1038	}
1039	else
1040	pCpu->fFlushAsidBeforeUse = true;
1041
1042	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1043	++pCpu->cTlbFlushes;
1044
1045	return VINF_SUCCESS;
1046	}
1047
1048
1049	/**
1050	* Deactivates VT-x on the current CPU.
1051	*
1052	* @returns VBox status code.
1053	* @param pCpu Pointer to the global CPU info struct.
1054	* @param pvCpuPage Pointer to the VMXON region.
1055	* @param HCPhysCpuPage Physical address of the VMXON region.
1056	*
1057	* @remarks This function should never be called when SUPR0EnableVTx() or
1058	* similar was used to enable VT-x on the host.
1059	*/
1060	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1061	{
1062	NOREF(pCpu);
1063	NOREF(pvCpuPage);
1064	NOREF(HCPhysCpuPage);
1065
1066	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067	return hmR0VmxLeaveRootMode();
1068	}
1069
1070
1071	/**
1072	* Sets the permission bits for the specified MSR in the MSR bitmap.
1073	*
1074	* @param pVCpu Pointer to the VMCPU.
1075	* @param uMSR The MSR value.
1076	* @param enmRead Whether reading this MSR causes a VM-exit.
1077	* @param enmWrite Whether writing this MSR causes a VM-exit.
1078	*/
1079	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1080	{
1081	int32_t iBit;
1082	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1083
1084	/*
1085	* Layout:
1086	* 0x000 - 0x3ff - Low MSR read bits
1087	* 0x400 - 0x7ff - High MSR read bits
1088	* 0x800 - 0xbff - Low MSR write bits
1089	* 0xc00 - 0xfff - High MSR write bits
1090	*/
1091	if (uMsr <= 0x00001FFF)
1092	iBit = uMsr;
1093	else if ( uMsr >= 0xC0000000
1094	&& uMsr <= 0xC0001FFF)
1095	{
1096	iBit = (uMsr - 0xC0000000);
1097	pbMsrBitmap += 0x400;
1098	}
1099	else
1100	{
1101	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1102	return;
1103	}
1104
1105	Assert(iBit <= 0x1fff);
1106	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1107	ASMBitSet(pbMsrBitmap, iBit);
1108	else
1109	ASMBitClear(pbMsrBitmap, iBit);
1110
1111	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1112	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1113	else
1114	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1115	}
1116
1117
1118	/**
1119	* Flushes the TLB using EPT.
1120	*
1121	* @returns VBox status code.
1122	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1123	* enmFlush).
1124	* @param enmFlush Type of flush.
1125	*
1126	* @remarks Caller is responsible for making sure this function is called only
1127	* when NestedPaging is supported and providing @a enmFlush that is
1128	* supported by the CPU.
1129	*/
1130	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1131	{
1132	uint64_t au64Descriptor[2];
1133	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1134	au64Descriptor[0] = 0;
1135	else
1136	{
1137	Assert(pVCpu);
1138	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1139	}
1140	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1141
1142	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1143	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1144	rc));
1145	if ( RT_SUCCESS(rc)
1146	&& pVCpu)
1147	{
1148	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1149	}
1150	}
1151
1152
1153	/**
1154	* Flushes the TLB using VPID.
1155	*
1156	* @returns VBox status code.
1157	* @param pVM Pointer to the VM.
1158	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1159	* enmFlush).
1160	* @param enmFlush Type of flush.
1161	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1162	* on @a enmFlush).
1163	*/
1164	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1165	{
1166	AssertPtr(pVM);
1167	Assert(pVM->hm.s.vmx.fVpid);
1168
1169	uint64_t au64Descriptor[2];
1170	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1171	{
1172	au64Descriptor[0] = 0;
1173	au64Descriptor[1] = 0;
1174	}
1175	else
1176	{
1177	AssertPtr(pVCpu);
1178	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1179	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1180	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1181	au64Descriptor[1] = GCPtr;
1182	}
1183
1184	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1185	AssertMsg(rc == VINF_SUCCESS,
1186	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1187	if ( RT_SUCCESS(rc)
1188	&& pVCpu)
1189	{
1190	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1191	}
1192	}
1193
1194
1195	/**
1196	* Invalidates a guest page by guest virtual address. Only relevant for
1197	* EPT/VPID, otherwise there is nothing really to invalidate.
1198	*
1199	* @returns VBox status code.
1200	* @param pVM Pointer to the VM.
1201	* @param pVCpu Pointer to the VMCPU.
1202	* @param GCVirt Guest virtual address of the page to invalidate.
1203	*/
1204	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1205	{
1206	AssertPtr(pVM);
1207	AssertPtr(pVCpu);
1208	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1209
1210	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1211	if (!fFlushPending)
1212	{
1213	/*
1214	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1215	* See @bugref{6043} and @bugref{6177}.
1216	*
1217	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1218	* function maybe called in a loop with individual addresses.
1219	*/
1220	if (pVM->hm.s.vmx.fVpid)
1221	{
1222	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1223	{
1224	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1225	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1226	}
1227	else
1228	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1229	}
1230	else if (pVM->hm.s.fNestedPaging)
1231	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1232	}
1233
1234	return VINF_SUCCESS;
1235	}
1236
1237
1238	/**
1239	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1240	* otherwise there is nothing really to invalidate.
1241	*
1242	* @returns VBox status code.
1243	* @param pVM Pointer to the VM.
1244	* @param pVCpu Pointer to the VMCPU.
1245	* @param GCPhys Guest physical address of the page to invalidate.
1246	*/
1247	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1248	{
1249	LogFlowFunc(("%RGp\n", GCPhys));
1250
1251	/*
1252	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1253	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1254	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1255	*/
1256	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1257	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1258	return VINF_SUCCESS;
1259	}
1260
1261
1262	/**
1263	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1264	* case where neither EPT nor VPID is supported by the CPU.
1265	*
1266	* @param pVM Pointer to the VM.
1267	* @param pVCpu Pointer to the VMCPU.
1268	* @param pCpu Pointer to the global HM struct.
1269	*
1270	* @remarks Called with interrupts disabled.
1271	*/
1272	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1273	{
1274	AssertPtr(pVCpu);
1275	AssertPtr(pCpu);
1276	NOREF(pVM);
1277
1278	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1279	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1280
1281	pVCpu->hm.s.TlbShootdown.cPages = 0;
1282	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1283	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1284	pVCpu->hm.s.fForceTLBFlush = false;
1285	return;
1286	}
1287
1288
1289	/**
1290	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1291	*
1292	* @param pVM Pointer to the VM.
1293	* @param pVCpu Pointer to the VMCPU.
1294	* @param pCpu Pointer to the global HM CPU struct.
1295	* @remarks All references to "ASID" in this function pertains to "VPID" in
1296	* Intel's nomenclature. The reason is, to avoid confusion in compare
1297	* statements since the host-CPU copies are named "ASID".
1298	*
1299	* @remarks Called with interrupts disabled.
1300	*/
1301	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1302	{
1303	#ifdef VBOX_WITH_STATISTICS
1304	bool fTlbFlushed = false;
1305	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1306	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1307	if (!fTlbFlushed) \
1308	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1309	} while (0)
1310	#else
1311	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1312	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1313	#endif
1314
1315	AssertPtr(pVM);
1316	AssertPtr(pCpu);
1317	AssertPtr(pVCpu);
1318	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1319	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1320	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1321
1322
1323	/*
1324	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1325	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1326	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1327	*/
1328	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1329	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1330	{
1331	++pCpu->uCurrentAsid;
1332	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1333	{
1334	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1335	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1336	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1337	}
1338
1339	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1340	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1341	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1342
1343	/*
1344	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1345	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1346	*/
1347	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1348	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1349	HMVMX_SET_TAGGED_TLB_FLUSHED();
1350	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1351	}
1352
1353	/* Check for explicit TLB shootdowns. */
1354	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1355	{
1356	/*
1357	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1358	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1359	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1360	* but not guest-physical mappings.
1361	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1362	*/
1363	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1364	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1365	HMVMX_SET_TAGGED_TLB_FLUSHED();
1366	}
1367
1368	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1369	* not be executed. See hmQueueInvlPage() where it is commented
1370	* out. Support individual entry flushing someday. */
1371	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1372	{
1373	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1374
1375	/*
1376	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1377	* as supported by the CPU.
1378	*/
1379	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1380	{
1381	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1382	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1383	}
1384	else
1385	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1386
1387	HMVMX_SET_TAGGED_TLB_FLUSHED();
1388	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1389	}
1390
1391	pVCpu->hm.s.TlbShootdown.cPages = 0;
1392	pVCpu->hm.s.fForceTLBFlush = false;
1393
1394	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1395
1396	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1397	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1398	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1399	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1400	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1401	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1402	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1403	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1404
1405	/* Update VMCS with the VPID. */
1406	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1407	AssertRC(rc);
1408
1409	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1410	}
1411
1412
1413	/**
1414	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1415	*
1416	* @returns VBox status code.
1417	* @param pVM Pointer to the VM.
1418	* @param pVCpu Pointer to the VMCPU.
1419	* @param pCpu Pointer to the global HM CPU struct.
1420	*
1421	* @remarks Called with interrupts disabled.
1422	*/
1423	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1424	{
1425	AssertPtr(pVM);
1426	AssertPtr(pVCpu);
1427	AssertPtr(pCpu);
1428	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1429	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1430
1431	/*
1432	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1433	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1434	*/
1435	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1436	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1437	{
1438	pVCpu->hm.s.fForceTLBFlush = true;
1439	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1440	}
1441
1442	/* Check for explicit TLB shootdown flushes. */
1443	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1444	{
1445	pVCpu->hm.s.fForceTLBFlush = true;
1446	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1447	}
1448
1449	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1450	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1451
1452	if (pVCpu->hm.s.fForceTLBFlush)
1453	{
1454	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1455	pVCpu->hm.s.fForceTLBFlush = false;
1456	}
1457	else
1458	{
1459	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1460	* not be executed. See hmQueueInvlPage() where it is commented
1461	* out. Support individual entry flushing someday. */
1462	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1463	{
1464	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1465	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1466	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1467	}
1468	else
1469	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1470	}
1471
1472	pVCpu->hm.s.TlbShootdown.cPages = 0;
1473	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1474	}
1475
1476
1477	/**
1478	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1479	*
1480	* @returns VBox status code.
1481	* @param pVM Pointer to the VM.
1482	* @param pVCpu Pointer to the VMCPU.
1483	* @param pCpu Pointer to the global HM CPU struct.
1484	*
1485	* @remarks Called with interrupts disabled.
1486	*/
1487	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1488	{
1489	AssertPtr(pVM);
1490	AssertPtr(pVCpu);
1491	AssertPtr(pCpu);
1492	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1493	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1494
1495	/*
1496	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1497	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1498	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1499	*/
1500	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1501	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1502	{
1503	pVCpu->hm.s.fForceTLBFlush = true;
1504	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1505	}
1506
1507	/* Check for explicit TLB shootdown flushes. */
1508	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1509	{
1510	/*
1511	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1512	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1513	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1514	*/
1515	pVCpu->hm.s.fForceTLBFlush = true;
1516	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1517	}
1518
1519	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1520	if (pVCpu->hm.s.fForceTLBFlush)
1521	{
1522	++pCpu->uCurrentAsid;
1523	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1524	{
1525	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1526	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1527	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1528	}
1529
1530	pVCpu->hm.s.fForceTLBFlush = false;
1531	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1532	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1533	if (pCpu->fFlushAsidBeforeUse)
1534	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1535	}
1536	else
1537	{
1538	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1539	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1540	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1541	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1542
1543	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1544	* not be executed. See hmQueueInvlPage() where it is commented
1545	* out. Support individual entry flushing someday. */
1546	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1547	{
1548	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1549	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1550	{
1551	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1552	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1553	}
1554	else
1555	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1556	}
1557	else
1558	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1559	}
1560
1561	pVCpu->hm.s.TlbShootdown.cPages = 0;
1562	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1563
1564	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1565	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1566	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1567	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1568	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1569	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1570
1571	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1572	AssertRC(rc);
1573	}
1574
1575
1576	/**
1577	* Flushes the guest TLB entry based on CPU capabilities.
1578	*
1579	* @param pVCpu Pointer to the VMCPU.
1580	* @param pCpu Pointer to the global HM CPU struct.
1581	*/
1582	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1583	{
1584	PVM pVM = pVCpu->CTX_SUFF(pVM);
1585	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1586	{
1587	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1588	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1589	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1590	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1591	default:
1592	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1593	break;
1594	}
1595	}
1596
1597
1598	/**
1599	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1600	* TLB entries from the host TLB before VM-entry.
1601	*
1602	* @returns VBox status code.
1603	* @param pVM Pointer to the VM.
1604	*/
1605	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1606	{
1607	/*
1608	* Determine optimal flush type for Nested Paging.
1609	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1610	* guest execution (see hmR3InitFinalizeR0()).
1611	*/
1612	if (pVM->hm.s.fNestedPaging)
1613	{
1614	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1615	{
1616	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1617	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1618	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1619	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1620	else
1621	{
1622	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1623	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1624	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1625	}
1626
1627	/* Make sure the write-back cacheable memory type for EPT is supported. */
1628	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1629	{
1630	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1631	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1632	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1633	}
1634	}
1635	else
1636	{
1637	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1638	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1639	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1640	}
1641	}
1642
1643	/*
1644	* Determine optimal flush type for VPID.
1645	*/
1646	if (pVM->hm.s.vmx.fVpid)
1647	{
1648	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1649	{
1650	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1651	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1652	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1653	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1654	else
1655	{
1656	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1657	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1658	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1659	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1660	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1661	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1662	pVM->hm.s.vmx.fVpid = false;
1663	}
1664	}
1665	else
1666	{
1667	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1668	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1669	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1670	pVM->hm.s.vmx.fVpid = false;
1671	}
1672	}
1673
1674	/*
1675	* Setup the handler for flushing tagged-TLBs.
1676	*/
1677	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1678	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1679	else if (pVM->hm.s.fNestedPaging)
1680	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1681	else if (pVM->hm.s.vmx.fVpid)
1682	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1683	else
1684	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1685	return VINF_SUCCESS;
1686	}
1687
1688
1689	/**
1690	* Sets up pin-based VM-execution controls in the VMCS.
1691	*
1692	* @returns VBox status code.
1693	* @param pVM Pointer to the VM.
1694	* @param pVCpu Pointer to the VMCPU.
1695	*/
1696	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1697	{
1698	AssertPtr(pVM);
1699	AssertPtr(pVCpu);
1700
1701	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1702	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1703
1704	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1705	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1706	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1707
1708	/* Enable the VMX preemption timer. */
1709	if (pVM->hm.s.vmx.fUsePreemptTimer)
1710	{
1711	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1712	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1713	}
1714
1715	if ((val & zap) != val)
1716	{
1717	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1718	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1719	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1720	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1721	}
1722
1723	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1724	AssertRCReturn(rc, rc);
1725
1726	/* Update VCPU with the currently set pin-based VM-execution controls. */
1727	pVCpu->hm.s.vmx.u32PinCtls = val;
1728	return rc;
1729	}
1730
1731
1732	/**
1733	* Sets up processor-based VM-execution controls in the VMCS.
1734	*
1735	* @returns VBox status code.
1736	* @param pVM Pointer to the VM.
1737	* @param pVMCPU Pointer to the VMCPU.
1738	*/
1739	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1740	{
1741	AssertPtr(pVM);
1742	AssertPtr(pVCpu);
1743
1744	int rc = VERR_INTERNAL_ERROR_5;
1745	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1746	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1747
1748	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1749	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1750	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1751	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1752	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1753	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1754	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1755
1756	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1757	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1758	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1759	{
1760	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1761	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1762	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1763	}
1764
1765	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1766	if (!pVM->hm.s.fNestedPaging)
1767	{
1768	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1769	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1770	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1771	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1772	}
1773
1774	/* Use TPR shadowing if supported by the CPU. */
1775	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1776	{
1777	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1778	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1779	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1780	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1781	AssertRCReturn(rc, rc);
1782
1783	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1784	/* CR8 writes causes a VM-exit based on TPR threshold. */
1785	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1786	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1787	}
1788	else
1789	{
1790	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1791	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1792	}
1793
1794	/* Use MSR-bitmaps if supported by the CPU. */
1795	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1796	{
1797	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1798
1799	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1800	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1801	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1802	AssertRCReturn(rc, rc);
1803
1804	/*
1805	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1806	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1807	*/
1808	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1809	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1810	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1811	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1813	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1814	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1815	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1817	}
1818
1819	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1820	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1821	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1822
1823	if ((val & zap) != val)
1824	{
1825	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1826	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1827	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1828	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1829	}
1830
1831	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1832	AssertRCReturn(rc, rc);
1833
1834	/* Update VCPU with the currently set processor-based VM-execution controls. */
1835	pVCpu->hm.s.vmx.u32ProcCtls = val;
1836
1837	/*
1838	* Secondary processor-based VM-execution controls.
1839	*/
1840	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1841	{
1842	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1843	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1844
1845	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1846	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1847
1848	if (pVM->hm.s.fNestedPaging)
1849	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1850	else
1851	{
1852	/*
1853	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1854	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1855	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1856	*/
1857	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1858	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1859	}
1860
1861	if (pVM->hm.s.vmx.fVpid)
1862	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1863
1864	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1865	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1866
1867	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1868	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1869	* done dynamically. */
1870	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1871	{
1872	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1873	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1874	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1875	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1876	AssertRCReturn(rc, rc);
1877	}
1878
1879	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1880	{
1881	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1882	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1883	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1884	}
1885
1886	if ((val & zap) != val)
1887	{
1888	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1889	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1890	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1891	}
1892
1893	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1894	AssertRCReturn(rc, rc);
1895
1896	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1897	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1898	}
1899	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1900	{
1901	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1902	"available\n"));
1903	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1904	}
1905
1906	return VINF_SUCCESS;
1907	}
1908
1909
1910	/**
1911	* Sets up miscellaneous (everything other than Pin & Processor-based
1912	* VM-execution) control fields in the VMCS.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	* @param pVCpu Pointer to the VMCPU.
1917	*/
1918	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1919	{
1920	AssertPtr(pVM);
1921	AssertPtr(pVCpu);
1922
1923	int rc = VERR_GENERAL_FAILURE;
1924
1925	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1926	#if 0
1927	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1928	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1929	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1930
1931	/*
1932	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1933	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1934	* We thus use the exception bitmap to control it rather than use both.
1935	*/
1936	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1937	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1938
1939	/** @todo Explore possibility of using IO-bitmaps. */
1940	/* All IO & IOIO instructions cause VM-exits. */
1941	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1942	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1943
1944	/* Initialize the MSR-bitmap area. */
1945	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1946	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1947	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1948	#endif
1949
1950	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1951	/* Setup MSR autoloading/storing. */
1952	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1953	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1954	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1955	AssertRCReturn(rc, rc);
1956	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1957	AssertRCReturn(rc, rc);
1958
1959	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1960	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1961	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1962	AssertRCReturn(rc, rc);
1963	#endif
1964
1965	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1966	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1967	AssertRCReturn(rc, rc);
1968
1969	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1970	#if 0
1971	/* Setup debug controls */
1972	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1973	AssertRCReturn(rc, rc);
1974	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1975	AssertRCReturn(rc, rc);
1976	#endif
1977
1978	return rc;
1979	}
1980
1981
1982	/**
1983	* Sets up the initial exception bitmap in the VMCS based on static conditions
1984	* (i.e. conditions that cannot ever change after starting the VM).
1985	*
1986	* @returns VBox status code.
1987	* @param pVM Pointer to the VM.
1988	* @param pVCpu Pointer to the VMCPU.
1989	*/
1990	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1991	{
1992	AssertPtr(pVM);
1993	AssertPtr(pVCpu);
1994
1995	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1996
1997	uint32_t u32XcptBitmap = 0;
1998
1999	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2000	if (!pVM->hm.s.fNestedPaging)
2001	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2002
2003	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2004	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2005	AssertRCReturn(rc, rc);
2006	return rc;
2007	}
2008
2009
2010	/**
2011	* Sets up the initial guest-state mask. The guest-state mask is consulted
2012	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2013	* for the nested virtualization case (as it would cause a VM-exit).
2014	*
2015	* @param pVCpu Pointer to the VMCPU.
2016	*/
2017	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2018	{
2019	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2020	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2021	return VINF_SUCCESS;
2022	}
2023
2024
2025	/**
2026	* Does per-VM VT-x initialization.
2027	*
2028	* @returns VBox status code.
2029	* @param pVM Pointer to the VM.
2030	*/
2031	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2032	{
2033	LogFlowFunc(("pVM=%p\n", pVM));
2034
2035	int rc = hmR0VmxStructsAlloc(pVM);
2036	if (RT_FAILURE(rc))
2037	{
2038	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2039	return rc;
2040	}
2041
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Does per-VM VT-x termination.
2048	*
2049	* @returns VBox status code.
2050	* @param pVM Pointer to the VM.
2051	*/
2052	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2053	{
2054	LogFlowFunc(("pVM=%p\n", pVM));
2055
2056	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2057	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2058	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2059	#endif
2060	hmR0VmxStructsFree(pVM);
2061	return VINF_SUCCESS;
2062	}
2063
2064
2065	/**
2066	* Sets up the VM for execution under VT-x.
2067	* This function is only called once per-VM during initialization.
2068	*
2069	* @returns VBox status code.
2070	* @param pVM Pointer to the VM.
2071	*/
2072	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2073	{
2074	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2075	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2076
2077	LogFlowFunc(("pVM=%p\n", pVM));
2078
2079	/*
2080	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2081	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2082	*/
2083	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2084	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2085	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2086	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2087	{
2088	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2089	return VERR_INTERNAL_ERROR;
2090	}
2091
2092	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2093	/*
2094	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2095	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2096	*/
2097	if ( pVM->hm.s.fAllow64BitGuests
2098	&& !HMVMX_IS_64BIT_HOST_MODE())
2099	{
2100	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2101	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2102	}
2103	#endif
2104
2105	/* Initialize these always, see hmR3InitFinalizeR0().*/
2106	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2107	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2108
2109	/* Setup the tagged-TLB flush handlers. */
2110	int rc = hmR0VmxSetupTaggedTlb(pVM);
2111	if (RT_FAILURE(rc))
2112	{
2113	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2114	return rc;
2115	}
2116
2117	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2118	{
2119	PVMCPU pVCpu = &pVM->aCpus[i];
2120	AssertPtr(pVCpu);
2121	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2122
2123	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2124	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2125
2126	/* Set revision dword at the beginning of the VMCS structure. */
2127	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2128
2129	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2130	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2131	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2132	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2133
2134	/* Load this VMCS as the current VMCS. */
2135	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2136	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2137	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2138
2139	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2140	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2141	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2142
2143	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2144	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2145	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2146
2147	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2148	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2149	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2150
2151	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2152	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2153	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2154
2155	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2156	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2157	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2158
2159	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2160	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2161	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2162	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2163	#endif
2164
2165	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2166	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2167	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2168	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2169
2170	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2171
2172	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2173	}
2174
2175	return VINF_SUCCESS;
2176	}
2177
2178
2179	/**
2180	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2181	* the VMCS.
2182	*
2183	* @returns VBox status code.
2184	* @param pVM Pointer to the VM.
2185	* @param pVCpu Pointer to the VMCPU.
2186	*/
2187	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2188	{
2189	RTCCUINTREG uReg = ASMGetCR0();
2190	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2191	AssertRCReturn(rc, rc);
2192
2193	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2194	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2195	if (HMVMX_IS_64BIT_HOST_MODE())
2196	{
2197	uint64_t uRegCR3 = HMR0Get64bitCR3();
2198	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2199	}
2200	else
2201	#endif
2202	{
2203	uReg = ASMGetCR3();
2204	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2205	}
2206	AssertRCReturn(rc, rc);
2207
2208	uReg = ASMGetCR4();
2209	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2210	AssertRCReturn(rc, rc);
2211	return rc;
2212	}
2213
2214
2215	/**
2216	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2217	* the host-state area in the VMCS.
2218	*
2219	* @returns VBox status code.
2220	* @param pVM Pointer to the VM.
2221	* @param pVCpu Pointer to the VMCPU.
2222	*/
2223	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2224	{
2225	int rc = VERR_INTERNAL_ERROR_5;
2226	RTSEL uSelDS = 0;
2227	RTSEL uSelES = 0;
2228	RTSEL uSelFS = 0;
2229	RTSEL uSelGS = 0;
2230	RTSEL uSelTR = 0;
2231
2232	/*
2233	* Host DS, ES, FS and GS segment registers.
2234	*/
2235	#if HC_ARCH_BITS == 64
2236	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2237	uSelDS = ASMGetDS();
2238	uSelES = ASMGetES();
2239	uSelFS = ASMGetFS();
2240	uSelGS = ASMGetGS();
2241	#endif
2242
2243	/*
2244	* Host CS and SS segment registers.
2245	*/
2246	RTSEL uSelCS;
2247	RTSEL uSelSS;
2248	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2249	if (HMVMX_IS_64BIT_HOST_MODE())
2250	{
2251	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2252	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2253	}
2254	else
2255	{
2256	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2257	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2258	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2259	}
2260	#else
2261	uSelCS = ASMGetCS();
2262	uSelSS = ASMGetSS();
2263	#endif
2264
2265	/*
2266	* Host TR segment register.
2267	*/
2268	uSelTR = ASMGetTR();
2269
2270	#if HC_ARCH_BITS == 64
2271	/*
2272	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2273	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2274	*/
2275	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2276	{
2277	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2278	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2279	uSelDS = 0;
2280	}
2281	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2282	{
2283	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2284	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2285	uSelES = 0;
2286	}
2287	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2288	{
2289	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2290	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2291	uSelFS = 0;
2292	}
2293	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2294	{
2295	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2296	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2297	uSelGS = 0;
2298	}
2299	#endif
2300
2301	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2302	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2303	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2304	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2305	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2306	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2307	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2308	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2309	Assert(uSelCS);
2310	Assert(uSelTR);
2311
2312	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2313	#if 0
2314	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2315	Assert(uSelSS != 0);
2316	#endif
2317
2318	/* Write these host selector fields into the host-state area in the VMCS. */
2319	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2320	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2321	#if HC_ARCH_BITS == 64
2322	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2323	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2324	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2325	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2326	#endif
2327	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2328
2329	/*
2330	* Host GDTR and IDTR.
2331	*/
2332	RTGDTR Gdtr;
2333	RT_ZERO(Gdtr);
2334	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2335	if (HMVMX_IS_64BIT_HOST_MODE())
2336	{
2337	X86XDTR64 Gdtr64;
2338	X86XDTR64 Idtr64;
2339	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2340	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2341	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2342
2343	Gdtr.cbGdt = Gdtr64.cb;
2344	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2345	}
2346	else
2347	#endif
2348	{
2349	RTIDTR Idtr;
2350	ASMGetGDTR(&Gdtr);
2351	ASMGetIDTR(&Idtr);
2352	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2353	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2354
2355	#if HC_ARCH_BITS == 64
2356	/*
2357	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2358	* maximum limit (0xffff) on every VM-exit.
2359	*/
2360	if (Gdtr.cbGdt != 0xffff)
2361	{
2362	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2363	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2364	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2365	}
2366
2367	/*
2368	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2369	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2370	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2371	*/
2372	if (Idtr.cbIdt < 0x0fff)
2373	{
2374	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2375	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2376	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2377	}
2378	#endif
2379	}
2380
2381	/*
2382	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2383	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2384	*/
2385	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2386	{
2387	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2388	return VERR_VMX_INVALID_HOST_STATE;
2389	}
2390
2391	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2392	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2393	if (HMVMX_IS_64BIT_HOST_MODE())
2394	{
2395	/* We need the 64-bit TR base for hybrid darwin. */
2396	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2397	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2398	}
2399	else
2400	#endif
2401	{
2402	uintptr_t uTRBase;
2403	#if HC_ARCH_BITS == 64
2404	uTRBase = X86DESC64_BASE(pDesc);
2405
2406	/*
2407	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2408	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2409	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2410	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2411	*
2412	* [1] See Intel spec. 3.5 "System Descriptor Types".
2413	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2414	*/
2415	Assert(pDesc->System.u4Type == 11);
2416	if ( pDesc->System.u16LimitLow != 0x67
2417	\|\| pDesc->System.u4LimitHigh)
2418	{
2419	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2420	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2421
2422	/* Store the GDTR here as we need it while restoring TR. */
2423	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2424	}
2425	#else
2426	uTRBase = X86DESC_BASE(pDesc);
2427	#endif
2428	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2429	}
2430	AssertRCReturn(rc, rc);
2431
2432	/*
2433	* Host FS base and GS base.
2434	*/
2435	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2436	if (HMVMX_IS_64BIT_HOST_MODE())
2437	{
2438	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2439	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2440	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2441	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2442
2443	# if HC_ARCH_BITS == 64
2444	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2445	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2446	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2447	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2448	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2449	# endif
2450	}
2451	#endif
2452	return rc;
2453	}
2454
2455
2456	/**
2457	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2458	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2459	* the host after every successful VM exit.
2460	*
2461	* @returns VBox status code.
2462	* @param pVM Pointer to the VM.
2463	* @param pVCpu Pointer to the VMCPU.
2464	*/
2465	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2466	{
2467	AssertPtr(pVCpu);
2468	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2469
2470	int rc = VINF_SUCCESS;
2471	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2472	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2473	uint32_t cHostMsrs = 0;
2474	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2475
2476	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2477	{
2478	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2479
2480	# if HC_ARCH_BITS == 64
2481	/* Paranoia. 64-bit code requires these bits to be set always. */
2482	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2483
2484	/*
2485	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2486	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2487	* some reason (e.g. allow transparent reads) we would activate the code below.
2488	*/
2489	# if 0
2490	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2491	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2492	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2493	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2494	if (CPUMIsGuestInLongMode(pVCpu))
2495	{
2496	uint64_t u64GuestEfer;
2497	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2498	AssertRC(rc);
2499
2500	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2501	{
2502	pHostMsr->u32Msr = MSR_K6_EFER;
2503	pHostMsr->u32Reserved = 0;
2504	pHostMsr->u64Value = u64HostEfer;
2505	pHostMsr++; cHostMsrs++;
2506	}
2507	}
2508	# endif
2509	# else /* HC_ARCH_BITS != 64 */
2510	pHostMsr->u32Msr = MSR_K6_EFER;
2511	pHostMsr->u32Reserved = 0;
2512	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2513	if (CPUMIsGuestInLongMode(pVCpu))
2514	{
2515	/* Must match the EFER value in our 64 bits switcher. */
2516	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2517	}
2518	else
2519	# endif
2520	pHostMsr->u64Value = u64HostEfer;
2521	pHostMsr++; cHostMsrs++;
2522	# endif /* HC_ARCH_BITS == 64 */
2523	}
2524
2525	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2526	if (HMVMX_IS_64BIT_HOST_MODE())
2527	{
2528	pHostMsr->u32Msr = MSR_K6_STAR;
2529	pHostMsr->u32Reserved = 0;
2530	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2531	pHostMsr++; cHostMsrs++;
2532	pHostMsr->u32Msr = MSR_K8_LSTAR;
2533	pHostMsr->u32Reserved = 0;
2534	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2535	pHostMsr++; cHostMsrs++;
2536	pHostMsr->u32Msr = MSR_K8_SF_MASK;
2537	pHostMsr->u32Reserved = 0;
2538	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2539	pHostMsr++; cHostMsrs++;
2540	pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2541	pHostMsr->u32Reserved = 0;
2542	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2543	pHostMsr++; cHostMsrs++;
2544	}
2545	# endif
2546
2547	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2548	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2549	{
2550	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2551	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2552	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2553	}
2554
2555	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2556	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2557
2558	/*
2559	* Host Sysenter MSRs.
2560	*/
2561	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2562	AssertRCReturn(rc, rc);
2563	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2564	if (HMVMX_IS_64BIT_HOST_MODE())
2565	{
2566	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2567	AssertRCReturn(rc, rc);
2568	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2569	}
2570	else
2571	{
2572	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2573	AssertRCReturn(rc, rc);
2574	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2575	}
2576	#elif HC_ARCH_BITS == 32
2577	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2578	AssertRCReturn(rc, rc);
2579	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2580	#else
2581	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2582	AssertRCReturn(rc, rc);
2583	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2584	#endif
2585	AssertRCReturn(rc, rc);
2586
2587	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2588	* hmR0VmxSetupExitCtls() !! */
2589	return rc;
2590	}
2591
2592
2593	/**
2594	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2595	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2596	* controls".
2597	*
2598	* @returns VBox status code.
2599	* @param pVCpu Pointer to the VMCPU.
2600	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2601	* out-of-sync. Make sure to update the required fields
2602	* before using them.
2603	*
2604	* @remarks No-long-jump zone!!!
2605	*/
2606	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2607	{
2608	int rc = VINF_SUCCESS;
2609	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2610	{
2611	PVM pVM = pVCpu->CTX_SUFF(pVM);
2612	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2613	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2614
2615	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2616	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2617
2618	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2619	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2620	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2621	else
2622	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2623
2624	/*
2625	* The following should not be set (since we're not in SMM mode):
2626	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2627	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2628	*/
2629
2630	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2631	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2632	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2633
2634	if ((val & zap) != val)
2635	{
2636	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2637	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2638	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2639	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2640	}
2641
2642	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2643	AssertRCReturn(rc, rc);
2644
2645	/* Update VCPU with the currently set VM-exit controls. */
2646	pVCpu->hm.s.vmx.u32EntryCtls = val;
2647	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2648	}
2649	return rc;
2650	}
2651
2652
2653	/**
2654	* Sets up the VM-exit controls in the VMCS.
2655	*
2656	* @returns VBox status code.
2657	* @param pVM Pointer to the VM.
2658	* @param pVCpu Pointer to the VMCPU.
2659	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2660	* out-of-sync. Make sure to update the required fields
2661	* before using them.
2662	*
2663	* @remarks requires EFER.
2664	*/
2665	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2666	{
2667	int rc = VINF_SUCCESS;
2668	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2669	{
2670	PVM pVM = pVCpu->CTX_SUFF(pVM);
2671	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2672	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2673
2674	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2675	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2676
2677	/*
2678	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2679	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2680	*/
2681	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2682	if (HMVMX_IS_64BIT_HOST_MODE())
2683	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2684	else
2685	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2686	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2687	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2688	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2689	else
2690	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2691	#endif
2692
2693	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2694	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2695
2696	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2697	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2698	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2699	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2700	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2701
2702	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2703	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2704
2705	if ((val & zap) != val)
2706	{
2707	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2708	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2709	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2710	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2711	}
2712
2713	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2714	AssertRCReturn(rc, rc);
2715
2716	/* Update VCPU with the currently set VM-exit controls. */
2717	pVCpu->hm.s.vmx.u32ExitCtls = val;
2718	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2719	}
2720	return rc;
2721	}
2722
2723
2724	/**
2725	* Loads the guest APIC and related state.
2726	*
2727	* @returns VBox status code.
2728	* @param pVM Pointer to the VM.
2729	* @param pVCpu Pointer to the VMCPU.
2730	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2731	* out-of-sync. Make sure to update the required fields
2732	* before using them.
2733	*/
2734	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2735	{
2736	int rc = VINF_SUCCESS;
2737	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2738	{
2739	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2740	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2741	{
2742	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2743
2744	bool fPendingIntr = false;
2745	uint8_t u8Tpr = 0;
2746	uint8_t u8PendingIntr = 0;
2747	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2748	AssertRCReturn(rc, rc);
2749
2750	/*
2751	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2752	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2753	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2754	* the interrupt when we VM-exit for other reasons.
2755	*/
2756	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2757	uint32_t u32TprThreshold = 0;
2758	if (fPendingIntr)
2759	{
2760	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2761	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2762	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2763	if (u8PendingPriority <= u8TprPriority)
2764	u32TprThreshold = u8PendingPriority;
2765	else
2766	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2767	}
2768	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2769
2770	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2771	AssertRCReturn(rc, rc);
2772	}
2773
2774	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2775	}
2776	return rc;
2777	}
2778
2779
2780	/**
2781	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2782	*
2783	* @returns Guest's interruptibility-state.
2784	* @param pVCpu Pointer to the VMCPU.
2785	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2786	* out-of-sync. Make sure to update the required fields
2787	* before using them.
2788	*
2789	* @remarks No-long-jump zone!!!
2790	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2791	*/
2792	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2793	{
2794	/*
2795	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2796	* inhibit interrupts or clear any existing interrupt-inhibition.
2797	*/
2798	uint32_t uIntrState = 0;
2799	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2800	{
2801	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2802	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2803	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2804	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2805	{
2806	/*
2807	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2808	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2809	*/
2810	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2811	}
2812	else if (pMixedCtx->eflags.Bits.u1IF)
2813	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2814	else
2815	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2816	}
2817	return uIntrState;
2818	}
2819
2820
2821	/**
2822	* Loads the guest's interruptibility-state into the guest-state area in the
2823	* VMCS.
2824	*
2825	* @returns VBox status code.
2826	* @param pVCpu Pointer to the VMCPU.
2827	* @param uIntrState The interruptibility-state to set.
2828	*/
2829	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2830	{
2831	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2832	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2833	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2834	AssertRCReturn(rc, rc);
2835	return rc;
2836	}
2837
2838
2839	/**
2840	* Loads the guest's RIP into the guest-state area in the VMCS.
2841	*
2842	* @returns VBox status code.
2843	* @param pVCpu Pointer to the VMCPU.
2844	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2845	* out-of-sync. Make sure to update the required fields
2846	* before using them.
2847	*
2848	* @remarks No-long-jump zone!!!
2849	*/
2850	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2851	{
2852	int rc = VINF_SUCCESS;
2853	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2854	{
2855	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2856	AssertRCReturn(rc, rc);
2857	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2858	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2859	}
2860	return rc;
2861	}
2862
2863
2864	/**
2865	* Loads the guest's RSP into the guest-state area in the VMCS.
2866	*
2867	* @returns VBox status code.
2868	* @param pVCpu Pointer to the VMCPU.
2869	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2870	* out-of-sync. Make sure to update the required fields
2871	* before using them.
2872	*
2873	* @remarks No-long-jump zone!!!
2874	*/
2875	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2876	{
2877	int rc = VINF_SUCCESS;
2878	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2879	{
2880	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2881	AssertRCReturn(rc, rc);
2882	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2883	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2884	}
2885	return rc;
2886	}
2887
2888
2889	/**
2890	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2891	*
2892	* @returns VBox status code.
2893	* @param pVCpu Pointer to the VMCPU.
2894	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2895	* out-of-sync. Make sure to update the required fields
2896	* before using them.
2897	*
2898	* @remarks No-long-jump zone!!!
2899	*/
2900	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2901	{
2902	int rc = VINF_SUCCESS;
2903	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2904	{
2905	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2906	Let us assert it as such and use 32-bit VMWRITE. */
2907	Assert(!(pMixedCtx->rflags.u64 >> 32));
2908	X86EFLAGS Eflags = pMixedCtx->eflags;
2909	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2910	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2911
2912	/*
2913	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2914	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2915	*/
2916	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2917	{
2918	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2919	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2920	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2921	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2922	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2923	}
2924
2925	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2926	AssertRCReturn(rc, rc);
2927
2928	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2929	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2930	}
2931	return rc;
2932	}
2933
2934
2935	/**
2936	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2937	*
2938	* @returns VBox status code.
2939	* @param pVCpu Pointer to the VMCPU.
2940	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2941	* out-of-sync. Make sure to update the required fields
2942	* before using them.
2943	*
2944	* @remarks No-long-jump zone!!!
2945	*/
2946	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2947	{
2948	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2949	AssertRCReturn(rc, rc);
2950	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2951	AssertRCReturn(rc, rc);
2952	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2953	AssertRCReturn(rc, rc);
2954	return rc;
2955	}
2956
2957
2958	/**
2959	* Loads the guest CR0 control register into the guest-state area in the VMCS.
2960	* CR0 is partially shared with the host and we have to consider the FPU bits.
2961	*
2962	* @returns VBox status code.
2963	* @param pVM Pointer to the VM.
2964	* @param pVCpu Pointer to the VMCPU.
2965	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2966	* out-of-sync. Make sure to update the required fields
2967	* before using them.
2968	*
2969	* @remarks No-long-jump zone!!!
2970	*/
2971	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2972	{
2973	/*
2974	* Guest CR0.
2975	* Guest FPU.
2976	*/
2977	int rc = VINF_SUCCESS;
2978	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2979	{
2980	Assert(!(pMixedCtx->cr0 >> 32));
2981	uint32_t u32GuestCR0 = pMixedCtx->cr0;
2982	PVM pVM = pVCpu->CTX_SUFF(pVM);
2983
2984	/* The guest's view (read access) of its CR0 is unblemished. */
2985	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2986	AssertRCReturn(rc, rc);
2987	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2988
2989	/* Setup VT-x's view of the guest CR0. */
2990	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2991	if (pVM->hm.s.fNestedPaging)
2992	{
2993	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
2994	{
2995	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2996	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2997	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2998	}
2999	else
3000	{
3001	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3002	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3003	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3004	}
3005
3006	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3007	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3008	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3009
3010	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3011	AssertRCReturn(rc, rc);
3012	}
3013	else
3014	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3015
3016	/*
3017	* Guest FPU bits.
3018	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3019	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3020	*/
3021	u32GuestCR0 \|= X86_CR0_NE;
3022	bool fInterceptNM = false;
3023	if (CPUMIsGuestFPUStateActive(pVCpu))
3024	{
3025	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3026	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3027	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3028	}
3029	else
3030	{
3031	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3032	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3033	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3034	}
3035
3036	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3037	bool fInterceptMF = false;
3038	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3039	fInterceptMF = true;
3040
3041	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3042	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3043	{
3044	Assert(PDMVmmDevHeapIsEnabled(pVM));
3045	Assert(pVM->hm.s.vmx.pRealModeTSS);
3046	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3047	fInterceptNM = true;
3048	fInterceptMF = true;
3049	}
3050	else
3051	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3052
3053	if (fInterceptNM)
3054	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3055	else
3056	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3057
3058	if (fInterceptMF)
3059	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3060	else
3061	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3062
3063	/* Additional intercepts for debugging, define these yourself explicitly. */
3064	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3065	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3066	\| RT_BIT(X86_XCPT_BP)
3067	\| RT_BIT(X86_XCPT_DB)
3068	\| RT_BIT(X86_XCPT_DE)
3069	\| RT_BIT(X86_XCPT_NM)
3070	\| RT_BIT(X86_XCPT_UD)
3071	\| RT_BIT(X86_XCPT_NP)
3072	\| RT_BIT(X86_XCPT_SS)
3073	\| RT_BIT(X86_XCPT_GP)
3074	\| RT_BIT(X86_XCPT_PF)
3075	\| RT_BIT(X86_XCPT_MF)
3076	;
3077	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3078	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3079	#endif
3080
3081	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3082
3083	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3084	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3085	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3086	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3087	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3088	else
3089	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3090
3091	u32GuestCR0 \|= uSetCR0;
3092	u32GuestCR0 &= uZapCR0;
3093	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3094
3095	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3096	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3097	AssertRCReturn(rc, rc);
3098	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3099	AssertRCReturn(rc, rc);
3100	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3101
3102	/*
3103	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3104	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3105	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3106	*/
3107	uint32_t u32CR0Mask = 0;
3108	u32CR0Mask = X86_CR0_PE
3109	\| X86_CR0_NE
3110	\| X86_CR0_WP
3111	\| X86_CR0_PG
3112	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3113	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3114	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3115	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3116	u32CR0Mask &= ~X86_CR0_PE;
3117	if (pVM->hm.s.fNestedPaging)
3118	u32CR0Mask &= ~X86_CR0_WP;
3119
3120	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3121	if (fInterceptNM)
3122	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3123	else
3124	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3125
3126	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3127	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3128	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3129	AssertRCReturn(rc, rc);
3130
3131	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3132	}
3133	return rc;
3134	}
3135
3136
3137	/**
3138	* Loads the guest control registers (CR3, CR4) into the guest-state area
3139	* in the VMCS.
3140	*
3141	* @returns VBox status code.
3142	* @param pVM Pointer to the VM.
3143	* @param pVCpu Pointer to the VMCPU.
3144	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3145	* out-of-sync. Make sure to update the required fields
3146	* before using them.
3147	*
3148	* @remarks No-long-jump zone!!!
3149	*/
3150	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3151	{
3152	int rc = VINF_SUCCESS;
3153	PVM pVM = pVCpu->CTX_SUFF(pVM);
3154
3155	/*
3156	* Guest CR2.
3157	* It's always loaded in the assembler code. Nothing to do here.
3158	*/
3159
3160	/*
3161	* Guest CR3.
3162	*/
3163	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3164	{
3165	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3166	if (pVM->hm.s.fNestedPaging)
3167	{
3168	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3169
3170	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3171	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3172	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3173	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3174
3175	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3176	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3177	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3178
3179	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3180	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3181	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3182	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3183
3184	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3185	AssertRCReturn(rc, rc);
3186	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3187
3188	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3189	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3190	{
3191	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3192	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3193	{
3194	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3195	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3196	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3197	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3198	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3199	}
3200
3201	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3202	have Unrestricted Execution to handle the guest when it's not using paging. */
3203	GCPhysGuestCR3 = pMixedCtx->cr3;
3204	}
3205	else
3206	{
3207	/*
3208	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3209	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3210	* EPT takes care of translating it to host-physical addresses.
3211	*/
3212	RTGCPHYS GCPhys;
3213	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3214	Assert(PDMVmmDevHeapIsEnabled(pVM));
3215
3216	/* We obtain it here every time as the guest could have relocated this PCI region. */
3217	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3218	AssertRCReturn(rc, rc);
3219
3220	GCPhysGuestCR3 = GCPhys;
3221	}
3222
3223	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3224	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3225	}
3226	else
3227	{
3228	/* Non-nested paging case, just use the hypervisor's CR3. */
3229	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3230
3231	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3232	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3233	}
3234	AssertRCReturn(rc, rc);
3235
3236	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3237	}
3238
3239	/*
3240	* Guest CR4.
3241	*/
3242	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3243	{
3244	Assert(!(pMixedCtx->cr4 >> 32));
3245	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3246
3247	/* The guest's view of its CR4 is unblemished. */
3248	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3249	AssertRCReturn(rc, rc);
3250	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3251
3252	/* Setup VT-x's view of the guest CR4. */
3253	/*
3254	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3255	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3256	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3257	*/
3258	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3259	{
3260	Assert(pVM->hm.s.vmx.pRealModeTSS);
3261	Assert(PDMVmmDevHeapIsEnabled(pVM));
3262	u32GuestCR4 &= ~X86_CR4_VME;
3263	}
3264
3265	if (pVM->hm.s.fNestedPaging)
3266	{
3267	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3268	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3269	{
3270	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3271	u32GuestCR4 \|= X86_CR4_PSE;
3272	/* Our identity mapping is a 32 bits page directory. */
3273	u32GuestCR4 &= ~X86_CR4_PAE;
3274	}
3275	/* else use guest CR4.*/
3276	}
3277	else
3278	{
3279	/*
3280	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3281	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3282	*/
3283	switch (pVCpu->hm.s.enmShadowMode)
3284	{
3285	case PGMMODE_REAL: /* Real-mode. */
3286	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3287	case PGMMODE_32_BIT: /* 32-bit paging. */
3288	{
3289	u32GuestCR4 &= ~X86_CR4_PAE;
3290	break;
3291	}
3292
3293	case PGMMODE_PAE: /* PAE paging. */
3294	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3295	{
3296	u32GuestCR4 \|= X86_CR4_PAE;
3297	break;
3298	}
3299
3300	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3301	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3302	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3303	break;
3304	#endif
3305	default:
3306	AssertFailed();
3307	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3308	}
3309	}
3310
3311	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3312	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3313	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3314	u32GuestCR4 \|= uSetCR4;
3315	u32GuestCR4 &= uZapCR4;
3316
3317	/* Write VT-x's view of the guest CR4 into the VMCS. */
3318	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3319	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3320	AssertRCReturn(rc, rc);
3321
3322	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3323	uint32_t u32CR4Mask = 0;
3324	u32CR4Mask = X86_CR4_VME
3325	\| X86_CR4_PAE
3326	\| X86_CR4_PGE
3327	\| X86_CR4_PSE
3328	\| X86_CR4_VMXE;
3329	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3330	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3331	AssertRCReturn(rc, rc);
3332
3333	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3334	}
3335	return rc;
3336	}
3337
3338
3339	/**
3340	* Loads the guest debug registers into the guest-state area in the VMCS.
3341	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3342	*
3343	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3344	*
3345	* @returns VBox status code.
3346	* @param pVCpu Pointer to the VMCPU.
3347	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3348	* out-of-sync. Make sure to update the required fields
3349	* before using them.
3350	*
3351	* @remarks No-long-jump zone!!!
3352	*/
3353	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3354	{
3355	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3356	return VINF_SUCCESS;
3357
3358	#ifdef VBOX_STRICT
3359	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3360	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3361	{
3362	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3363	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3364	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3365	}
3366	#endif
3367
3368	int rc;
3369	PVM pVM = pVCpu->CTX_SUFF(pVM);
3370	bool fInterceptDB = false;
3371	bool fInterceptMovDRx = false;
3372	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3373	{
3374	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3375	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3376	{
3377	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3378	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3379	AssertRCReturn(rc, rc);
3380	Assert(fInterceptDB == false);
3381	}
3382	else
3383	{
3384	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3385	pVCpu->hm.s.fClearTrapFlag = true;
3386	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3387	fInterceptDB = true;
3388	}
3389	}
3390
3391	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3392	{
3393	/*
3394	* Use the combined guest and host DRx values found in the hypervisor
3395	* register set because the debugger has breakpoints active or someone
3396	* is single stepping on the host side without a monitor trap flag.
3397	*
3398	* Note! DBGF expects a clean DR6 state before executing guest code.
3399	*/
3400	if (!CPUMIsHyperDebugStateActive(pVCpu))
3401	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3402	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3403	Assert(CPUMIsHyperDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
3404
3405	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3406	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3407	AssertRCReturn(rc, rc);
3408
3409	fInterceptDB = true;
3410	fInterceptMovDRx = true;
3411	}
3412	else
3413	{
3414	/*
3415	* If the guest has enabled debug registers, we need to load them prior to
3416	* executing guest code so they'll trigger at the right time.
3417	*/
3418	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3419	{
3420	if (!CPUMIsGuestDebugStateActive(pVCpu))
3421	{
3422	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3423	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3424	}
3425	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3426	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
3427	}
3428	/*
3429	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3430	* must intercept #DB in order to maintain a correct DR6 guest value.
3431	*/
3432	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3433	{
3434	fInterceptMovDRx = true;
3435	fInterceptDB = true;
3436	}
3437
3438	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3439	AssertRCReturn(rc, rc);
3440	}
3441
3442	/*
3443	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3444	*/
3445	if (fInterceptDB)
3446	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3447	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3448	{
3449	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3450	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3451	#endif
3452	}
3453	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3454	AssertRCReturn(rc, rc);
3455
3456	/*
3457	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3458	*/
3459	if (fInterceptMovDRx)
3460	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3461	else
3462	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3463	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3464	AssertRCReturn(rc, rc);
3465
3466	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3467	return VINF_SUCCESS;
3468	}
3469
3470
3471	#ifdef VBOX_STRICT
3472	/**
3473	* Strict function to validate segment registers.
3474	*
3475	* @remarks ASSUMES CR0 is up to date.
3476	*/
3477	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3478	{
3479	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3480	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3481	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3482	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3483	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3484	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3485	{
3486	/* Protected mode checks */
3487	/* CS */
3488	Assert(pCtx->cs.Attr.n.u1Present);
3489	Assert(!(pCtx->cs.Attr.u & 0xf00));
3490	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3491	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3492	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3493	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3494	\|\| (pCtx->cs.Attr.n.u1Granularity));
3495	/* CS cannot be loaded with NULL in protected mode. */
3496	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3497	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3498	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3499	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3500	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3501	else
3502	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3503	/* SS */
3504	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3505	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3506	if ( !(pCtx->cr0 & X86_CR0_PE)
3507	\|\| pCtx->cs.Attr.n.u4Type == 3)
3508	{
3509	Assert(!pCtx->ss.Attr.n.u2Dpl);
3510	}
3511	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3512	{
3513	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3514	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3515	Assert(pCtx->ss.Attr.n.u1Present);
3516	Assert(!(pCtx->ss.Attr.u & 0xf00));
3517	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3518	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3519	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3520	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3521	\|\| (pCtx->ss.Attr.n.u1Granularity));
3522	}
3523	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3524	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3525	{
3526	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3527	Assert(pCtx->ds.Attr.n.u1Present);
3528	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3529	Assert(!(pCtx->ds.Attr.u & 0xf00));
3530	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3531	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3532	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3533	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3534	\|\| (pCtx->ds.Attr.n.u1Granularity));
3535	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3536	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3537	}
3538	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3539	{
3540	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3541	Assert(pCtx->es.Attr.n.u1Present);
3542	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3543	Assert(!(pCtx->es.Attr.u & 0xf00));
3544	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3545	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3546	\|\| !(pCtx->es.Attr.n.u1Granularity));
3547	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3548	\|\| (pCtx->es.Attr.n.u1Granularity));
3549	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3550	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3551	}
3552	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3553	{
3554	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3555	Assert(pCtx->fs.Attr.n.u1Present);
3556	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3557	Assert(!(pCtx->fs.Attr.u & 0xf00));
3558	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3559	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3560	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3561	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3562	\|\| (pCtx->fs.Attr.n.u1Granularity));
3563	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3564	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3565	}
3566	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3567	{
3568	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3569	Assert(pCtx->gs.Attr.n.u1Present);
3570	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3571	Assert(!(pCtx->gs.Attr.u & 0xf00));
3572	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3573	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3574	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3575	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3576	\|\| (pCtx->gs.Attr.n.u1Granularity));
3577	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3578	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3579	}
3580	/* 64-bit capable CPUs. */
3581	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3582	Assert(!(pCtx->cs.u64Base >> 32));
3583	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3584	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3585	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3586	# endif
3587	}
3588	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3589	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3590	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3591	{
3592	/* Real and v86 mode checks. */
3593	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3594	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3595	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3596	{
3597	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3598	}
3599	else
3600	{
3601	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3602	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3603	}
3604
3605	/* CS */
3606	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3607	Assert(pCtx->cs.u32Limit == 0xffff);
3608	Assert(u32CSAttr == 0xf3);
3609	/* SS */
3610	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3611	Assert(pCtx->ss.u32Limit == 0xffff);
3612	Assert(u32SSAttr == 0xf3);
3613	/* DS */
3614	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3615	Assert(pCtx->ds.u32Limit == 0xffff);
3616	Assert(u32DSAttr == 0xf3);
3617	/* ES */
3618	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3619	Assert(pCtx->es.u32Limit == 0xffff);
3620	Assert(u32ESAttr == 0xf3);
3621	/* FS */
3622	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3623	Assert(pCtx->fs.u32Limit == 0xffff);
3624	Assert(u32FSAttr == 0xf3);
3625	/* GS */
3626	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3627	Assert(pCtx->gs.u32Limit == 0xffff);
3628	Assert(u32GSAttr == 0xf3);
3629	/* 64-bit capable CPUs. */
3630	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3631	Assert(!(pCtx->cs.u64Base >> 32));
3632	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3633	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3634	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3635	# endif
3636	}
3637	}
3638	#endif /* VBOX_STRICT */
3639
3640
3641	/**
3642	* Writes a guest segment register into the guest-state area in the VMCS.
3643	*
3644	* @returns VBox status code.
3645	* @param pVCpu Pointer to the VMCPU.
3646	* @param idxSel Index of the selector in the VMCS.
3647	* @param idxLimit Index of the segment limit in the VMCS.
3648	* @param idxBase Index of the segment base in the VMCS.
3649	* @param idxAccess Index of the access rights of the segment in the VMCS.
3650	* @param pSelReg Pointer to the segment selector.
3651	* @param pCtx Pointer to the guest-CPU context.
3652	*
3653	* @remarks No-long-jump zone!!!
3654	*/
3655	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3656	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3657	{
3658	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3659	AssertRCReturn(rc, rc);
3660	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3661	AssertRCReturn(rc, rc);
3662	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3663	AssertRCReturn(rc, rc);
3664
3665	uint32_t u32Access = pSelReg->Attr.u;
3666	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3667	{
3668	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3669	u32Access = 0xf3;
3670	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3671	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3672	}
3673	else
3674	{
3675	/*
3676	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3677	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3678	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3679	* loaded in protected-mode have their attribute as 0.
3680	*/
3681	if (!u32Access)
3682	u32Access = X86DESCATTR_UNUSABLE;
3683	}
3684
3685	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3686	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3687	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3688
3689	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3690	AssertRCReturn(rc, rc);
3691	return rc;
3692	}
3693
3694
3695	/**
3696	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3697	* into the guest-state area in the VMCS.
3698	*
3699	* @returns VBox status code.
3700	* @param pVM Pointer to the VM.
3701	* @param pVCPU Pointer to the VMCPU.
3702	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3703	* out-of-sync. Make sure to update the required fields
3704	* before using them.
3705	*
3706	* @remarks ASSUMES CR0 is up to date (strict builds validation).
3707	* @remarks No-long-jump zone!!!
3708	*/
3709	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3710	{
3711	int rc = VERR_INTERNAL_ERROR_5;
3712	PVM pVM = pVCpu->CTX_SUFF(pVM);
3713
3714	/*
3715	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3716	*/
3717	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3718	{
3719	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3720	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3721	{
3722	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3723	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3724	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3725	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3726	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3727	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3728	}
3729
3730	#ifdef VBOX_WITH_REM
3731	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3732	{
3733	Assert(pVM->hm.s.vmx.pRealModeTSS);
3734	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3735	if ( pVCpu->hm.s.vmx.fWasInRealMode
3736	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3737	{
3738	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3739	in real-mode (e.g. OpenBSD 4.0) */
3740	REMFlushTBs(pVM);
3741	Log4(("Load: Switch to protected mode detected!\n"));
3742	pVCpu->hm.s.vmx.fWasInRealMode = false;
3743	}
3744	}
3745	#endif
3746	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3747	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3748	AssertRCReturn(rc, rc);
3749	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3750	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3751	AssertRCReturn(rc, rc);
3752	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3753	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3754	AssertRCReturn(rc, rc);
3755	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3756	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3757	AssertRCReturn(rc, rc);
3758	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3759	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3760	AssertRCReturn(rc, rc);
3761	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3762	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3763	AssertRCReturn(rc, rc);
3764
3765	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3766	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3767	#ifdef VBOX_STRICT
3768	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3769	#endif
3770	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3771	}
3772
3773	/*
3774	* Guest TR.
3775	*/
3776	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3777	{
3778	/*
3779	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3780	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3781	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3782	*/
3783	uint16_t u16Sel = 0;
3784	uint32_t u32Limit = 0;
3785	uint64_t u64Base = 0;
3786	uint32_t u32AccessRights = 0;
3787
3788	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3789	{
3790	u16Sel = pMixedCtx->tr.Sel;
3791	u32Limit = pMixedCtx->tr.u32Limit;
3792	u64Base = pMixedCtx->tr.u64Base;
3793	u32AccessRights = pMixedCtx->tr.Attr.u;
3794	}
3795	else
3796	{
3797	Assert(pVM->hm.s.vmx.pRealModeTSS);
3798	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3799
3800	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3801	RTGCPHYS GCPhys;
3802	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3803	AssertRCReturn(rc, rc);
3804
3805	X86DESCATTR DescAttr;
3806	DescAttr.u = 0;
3807	DescAttr.n.u1Present = 1;
3808	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3809
3810	u16Sel = 0;
3811	u32Limit = HM_VTX_TSS_SIZE;
3812	u64Base = GCPhys; /* in real-mode phys = virt. */
3813	u32AccessRights = DescAttr.u;
3814	}
3815
3816	/* Validate. */
3817	Assert(!(u16Sel & RT_BIT(2)));
3818	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3819	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3820	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3821	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3822	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3823	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3824	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3825	Assert( (u32Limit & 0xfff) == 0xfff
3826	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3827	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3828	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3829
3830	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3831	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3832	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3833	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3834
3835	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3836	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3837	}
3838
3839	/*
3840	* Guest GDTR.
3841	*/
3842	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3843	{
3844	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3845	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3846
3847	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3848	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3849	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3850	}
3851
3852	/*
3853	* Guest LDTR.
3854	*/
3855	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3856	{
3857	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3858	uint32_t u32Access = 0;
3859	if (!pMixedCtx->ldtr.Attr.u)
3860	u32Access = X86DESCATTR_UNUSABLE;
3861	else
3862	u32Access = pMixedCtx->ldtr.Attr.u;
3863
3864	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3865	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3866	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3867	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3868
3869	/* Validate. */
3870	if (!(u32Access & X86DESCATTR_UNUSABLE))
3871	{
3872	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3873	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3874	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3875	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3876	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3877	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3878	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3879	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3880	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3881	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3882	}
3883
3884	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3885	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3886	}
3887
3888	/*
3889	* Guest IDTR.
3890	*/
3891	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3892	{
3893	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3894	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3895
3896	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3897	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3898	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3899	}
3900
3901	return VINF_SUCCESS;
3902	}
3903
3904
3905	/**
3906	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3907	* areas. These MSRs will automatically be loaded to the host CPU on every
3908	* successful VM entry and stored from the host CPU on every successful VM exit.
3909	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3910	*
3911	* @returns VBox status code.
3912	* @param pVCpu Pointer to the VMCPU.
3913	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3914	* out-of-sync. Make sure to update the required fields
3915	* before using them.
3916	*
3917	* @remarks No-long-jump zone!!!
3918	*/
3919	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3920	{
3921	AssertPtr(pVCpu);
3922	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3923
3924	/*
3925	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3926	*/
3927	int rc = VINF_SUCCESS;
3928	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3929	{
3930	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3931	PVM pVM = pVCpu->CTX_SUFF(pVM);
3932	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3933	uint32_t cGuestMsrs = 0;
3934
3935	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3936	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3937	* when the guest really is in 64-bit mode. */
3938	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3939	if (fSupportsLongMode)
3940	{
3941	pGuestMsr->u32Msr = MSR_K8_LSTAR;
3942	pGuestMsr->u32Reserved = 0;
3943	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3944	pGuestMsr++; cGuestMsrs++;
3945	pGuestMsr->u32Msr = MSR_K6_STAR;
3946	pGuestMsr->u32Reserved = 0;
3947	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3948	pGuestMsr++; cGuestMsrs++;
3949	pGuestMsr->u32Msr = MSR_K8_SF_MASK;
3950	pGuestMsr->u32Reserved = 0;
3951	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3952	pGuestMsr++; cGuestMsrs++;
3953	pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
3954	pGuestMsr->u32Reserved = 0;
3955	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3956	pGuestMsr++; cGuestMsrs++;
3957	}
3958
3959	/*
3960	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3961	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3962	*/
3963	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3964	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3965	{
3966	pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
3967	pGuestMsr->u32Reserved = 0;
3968	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3969	AssertRCReturn(rc, rc);
3970	pGuestMsr++; cGuestMsrs++;
3971	}
3972
3973	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3974	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
3975	{
3976	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3977	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
3978	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3979	}
3980
3981	/* Update the VCPU's copy of the guest MSR count. */
3982	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3983	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3984	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3985	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3986
3987	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3988	}
3989
3990	/*
3991	* Guest Sysenter MSRs.
3992	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3993	* VM-exits on WRMSRs for these MSRs.
3994	*/
3995	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3996	{
3997	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3998	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3999	}
4000	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4001	{
4002	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4003	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
4004	}
4005	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4006	{
4007	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4008	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
4009	}
4010
4011	return rc;
4012	}
4013
4014
4015	/**
4016	* Loads the guest activity state into the guest-state area in the VMCS.
4017	*
4018	* @returns VBox status code.
4019	* @param pVCpu Pointer to the VMCPU.
4020	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4021	* out-of-sync. Make sure to update the required fields
4022	* before using them.
4023	*
4024	* @remarks No-long-jump zone!!!
4025	*/
4026	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4027	{
4028	/** @todo See if we can make use of other states, e.g.
4029	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4030	int rc = VINF_SUCCESS;
4031	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4032	{
4033	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4034	AssertRCReturn(rc, rc);
4035	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4036	}
4037	return rc;
4038	}
4039
4040
4041	/**
4042	* Sets up the appropriate function to run guest code.
4043	*
4044	* @returns VBox status code.
4045	* @param pVCpu Pointer to the VMCPU.
4046	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4047	* out-of-sync. Make sure to update the required fields
4048	* before using them.
4049	*
4050	* @remarks No-long-jump zone!!!
4051	*/
4052	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4053	{
4054	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4055	{
4056	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4057	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4058	#endif
4059	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4060	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4061	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4062	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4063	{
4064	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4065	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS;
4066	}
4067	#else
4068	/* 64-bit host or hybrid host. */
4069	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4070	#endif
4071	}
4072	else
4073	{
4074	/* Guest is not in long mode, use the 32-bit handler. */
4075	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4076	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4077	{
4078	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4079	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS;
4080	}
4081	#else
4082	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4083	#endif
4084	}
4085	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4086	return VINF_SUCCESS;
4087	}
4088
4089
4090	/**
4091	* Wrapper for running the guest code in VT-x.
4092	*
4093	* @returns VBox strict status code.
4094	* @param pVM Pointer to the VM.
4095	* @param pVCpu Pointer to the VMCPU.
4096	* @param pCtx Pointer to the guest-CPU context.
4097	*
4098	* @remarks No-long-jump zone!!!
4099	*/
4100	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4101	{
4102	/*
4103	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4104	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4105	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4106	*/
4107	const bool fResumeVM = !!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4108	/** @todo Add stats for resume vs launch. */
4109	#ifdef VBOX_WITH_KERNEL_USING_XMM
4110	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4111	#else
4112	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4113	#endif
4114	}
4115
4116
4117	/**
4118	* Reports world-switch error and dumps some useful debug info.
4119	*
4120	* @param pVM Pointer to the VM.
4121	* @param pVCpu Pointer to the VMCPU.
4122	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4123	* @param pCtx Pointer to the guest-CPU context.
4124	* @param pVmxTransient Pointer to the VMX transient structure (only
4125	* exitReason updated).
4126	*/
4127	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4128	{
4129	Assert(pVM);
4130	Assert(pVCpu);
4131	Assert(pCtx);
4132	Assert(pVmxTransient);
4133	HMVMX_ASSERT_PREEMPT_SAFE();
4134
4135	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4136	switch (rcVMRun)
4137	{
4138	case VERR_VMX_INVALID_VMXON_PTR:
4139	AssertFailed();
4140	break;
4141	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4142	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4143	{
4144	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4145	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4146	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4147	AssertRC(rc);
4148
4149	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4150	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4151	Cannot do it here as we may have been long preempted. */
4152
4153	#ifdef VBOX_STRICT
4154	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4155	pVmxTransient->uExitReason));
4156	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4157	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4158	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4159	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4160	else
4161	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4162	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4163	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4164
4165	/* VMX control bits. */
4166	uint32_t u32Val;
4167	uint64_t u64Val;
4168	HMVMXHCUINTREG uHCReg;
4169	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4170	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4171	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4172	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4173	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4174	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4175	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4176	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4177	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4178	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4179	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4180	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4181	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4182	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4183	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4184	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4185	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4186	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4187	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4188	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4189	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4190	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4191	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4192	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4193	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4194	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4195	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4196	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4197	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4198	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4199	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4200	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4201	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4202	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4203	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4204	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4205	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4206	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4207	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4208	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4209	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4210	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4211
4212	/* Guest bits. */
4213	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4214	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4215	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4216	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4217	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4218	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4219	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4220	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4221
4222	/* Host bits. */
4223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4224	Log4(("Host CR0 %#RHr\n", uHCReg));
4225	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4226	Log4(("Host CR3 %#RHr\n", uHCReg));
4227	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4228	Log4(("Host CR4 %#RHr\n", uHCReg));
4229
4230	RTGDTR HostGdtr;
4231	PCX86DESCHC pDesc;
4232	ASMGetGDTR(&HostGdtr);
4233	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4234	Log4(("Host CS %#08x\n", u32Val));
4235	if (u32Val < HostGdtr.cbGdt)
4236	{
4237	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4238	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4239	}
4240
4241	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4242	Log4(("Host DS %#08x\n", u32Val));
4243	if (u32Val < HostGdtr.cbGdt)
4244	{
4245	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4246	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4247	}
4248
4249	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4250	Log4(("Host ES %#08x\n", u32Val));
4251	if (u32Val < HostGdtr.cbGdt)
4252	{
4253	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4254	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4255	}
4256
4257	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4258	Log4(("Host FS %#08x\n", u32Val));
4259	if (u32Val < HostGdtr.cbGdt)
4260	{
4261	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4262	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4263	}
4264
4265	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4266	Log4(("Host GS %#08x\n", u32Val));
4267	if (u32Val < HostGdtr.cbGdt)
4268	{
4269	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4270	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4271	}
4272
4273	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4274	Log4(("Host SS %#08x\n", u32Val));
4275	if (u32Val < HostGdtr.cbGdt)
4276	{
4277	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4278	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4279	}
4280
4281	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4282	Log4(("Host TR %#08x\n", u32Val));
4283	if (u32Val < HostGdtr.cbGdt)
4284	{
4285	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4286	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4287	}
4288
4289	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4290	Log4(("Host TR Base %#RHv\n", uHCReg));
4291	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4292	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4293	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4294	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4295	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4296	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4297	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4298	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4299	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4300	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4301	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4302	Log4(("Host RSP %#RHv\n", uHCReg));
4303	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4304	Log4(("Host RIP %#RHv\n", uHCReg));
4305	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4306	if (HMVMX_IS_64BIT_HOST_MODE())
4307	{
4308	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4309	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4310	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4311	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4312	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4313	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4314	}
4315	# endif
4316	#endif /* VBOX_STRICT */
4317	break;
4318	}
4319
4320	default:
4321	/* Impossible */
4322	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4323	break;
4324	}
4325	NOREF(pVM);
4326	}
4327
4328
4329	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4330	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4331	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4332	#endif
4333	#ifdef VBOX_STRICT
4334	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4335	{
4336	switch (idxField)
4337	{
4338	case VMX_VMCS_GUEST_RIP:
4339	case VMX_VMCS_GUEST_RSP:
4340	case VMX_VMCS_GUEST_SYSENTER_EIP:
4341	case VMX_VMCS_GUEST_SYSENTER_ESP:
4342	case VMX_VMCS_GUEST_GDTR_BASE:
4343	case VMX_VMCS_GUEST_IDTR_BASE:
4344	case VMX_VMCS_GUEST_CS_BASE:
4345	case VMX_VMCS_GUEST_DS_BASE:
4346	case VMX_VMCS_GUEST_ES_BASE:
4347	case VMX_VMCS_GUEST_FS_BASE:
4348	case VMX_VMCS_GUEST_GS_BASE:
4349	case VMX_VMCS_GUEST_SS_BASE:
4350	case VMX_VMCS_GUEST_LDTR_BASE:
4351	case VMX_VMCS_GUEST_TR_BASE:
4352	case VMX_VMCS_GUEST_CR3:
4353	return true;
4354	}
4355	return false;
4356	}
4357
4358	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4359	{
4360	switch (idxField)
4361	{
4362	/* Read-only fields. */
4363	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4364	return true;
4365	}
4366	/* Remaining readable fields should also be writable. */
4367	return hmR0VmxIsValidWriteField(idxField);
4368	}
4369	#endif /* VBOX_STRICT */
4370
4371
4372	/**
4373	* Executes the specified handler in 64-bit mode.
4374	*
4375	* @returns VBox status code.
4376	* @param pVM Pointer to the VM.
4377	* @param pVCpu Pointer to the VMCPU.
4378	* @param pCtx Pointer to the guest CPU context.
4379	* @param enmOp The operation to perform.
4380	* @param cbParam Number of parameters.
4381	* @param paParam Array of 32-bit parameters.
4382	*/
4383	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4384	uint32_t *paParam)
4385	{
4386	int rc, rc2;
4387	PHMGLOBALCPUINFO pCpu;
4388	RTHCPHYS HCPhysCpuPage;
4389	RTCCUINTREG uOldEflags;
4390
4391	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4392	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4393	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4394	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4395
4396	#ifdef VBOX_STRICT
4397	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4398	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4399
4400	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4401	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4402	#endif
4403
4404	/* Disable interrupts. */
4405	uOldEflags = ASMIntDisableFlags();
4406
4407	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4408	RTCPUID idHostCpu = RTMpCpuId();
4409	CPUMR0SetLApic(pVCpu, idHostCpu);
4410	#endif
4411
4412	pCpu = HMR0GetCurrentCpu();
4413	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4414
4415	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4416	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4417
4418	/* Leave VMX Root Mode. */
4419	VMXDisable();
4420
4421	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4422
4423	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4424	CPUMSetHyperEIP(pVCpu, enmOp);
4425	for (int i = (int)cbParam - 1; i >= 0; i--)
4426	CPUMPushHyper(pVCpu, paParam[i]);
4427
4428	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4429
4430	/* Call the switcher. */
4431	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4432	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4433
4434	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4435	/* Make sure the VMX instructions don't cause #UD faults. */
4436	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4437
4438	/* Re-enter VMX Root Mode */
4439	rc2 = VMXEnable(HCPhysCpuPage);
4440	if (RT_FAILURE(rc2))
4441	{
4442	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4443	ASMSetFlags(uOldEflags);
4444	return rc2;
4445	}
4446
4447	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4448	AssertRC(rc2);
4449	Assert(!(ASMGetFlags() & X86_EFL_IF));
4450	ASMSetFlags(uOldEflags);
4451	return rc;
4452	}
4453
4454
4455	/**
4456	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4457	* supporting 64-bit guests.
4458	*
4459	* @returns VBox status code.
4460	* @param fResume Whether to VMLAUNCH or VMRESUME.
4461	* @param pCtx Pointer to the guest-CPU context.
4462	* @param pCache Pointer to the VMCS cache.
4463	* @param pVM Pointer to the VM.
4464	* @param pVCpu Pointer to the VMCPU.
4465	*/
4466	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4467	{
4468	uint32_t aParam[6];
4469	PHMGLOBALCPUINFO pCpu = NULL;
4470	RTHCPHYS HCPhysCpuPage = 0;
4471	int rc = VERR_INTERNAL_ERROR_5;
4472
4473	pCpu = HMR0GetCurrentCpu();
4474	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4475
4476	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4477	pCache->uPos = 1;
4478	pCache->interPD = PGMGetInterPaeCR3(pVM);
4479	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4480	#endif
4481
4482	#ifdef VBOX_STRICT
4483	pCache->TestIn.HCPhysCpuPage = 0;
4484	pCache->TestIn.HCPhysVmcs = 0;
4485	pCache->TestIn.pCache = 0;
4486	pCache->TestOut.HCPhysVmcs = 0;
4487	pCache->TestOut.pCache = 0;
4488	pCache->TestOut.pCtx = 0;
4489	pCache->TestOut.eflags = 0;
4490	#endif
4491
4492	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4493	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4494	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4495	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4496	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4497	aParam[5] = 0;
4498
4499	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4500	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4501	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4502	#endif
4503	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4504
4505	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4506	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4507	Assert(pCtx->dr[4] == 10);
4508	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4509	#endif
4510
4511	#ifdef VBOX_STRICT
4512	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4513	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4514	pVCpu->hm.s.vmx.HCPhysVmcs));
4515	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4516	pCache->TestOut.HCPhysVmcs));
4517	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4518	pCache->TestOut.pCache));
4519	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4520	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4521	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4522	pCache->TestOut.pCtx));
4523	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4524	#endif
4525	return rc;
4526	}
4527
4528
4529	/**
4530	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4531	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4532	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4533	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4534	*
4535	* @returns VBox status code.
4536	* @param pVM Pointer to the VM.
4537	* @param pVCpu Pointer to the VMCPU.
4538	*/
4539	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4540	{
4541	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4542	{ \
4543	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4544	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4545	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4546	++cReadFields; \
4547	}
4548
4549	AssertPtr(pVM);
4550	AssertPtr(pVCpu);
4551	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4552	uint32_t cReadFields = 0;
4553
4554	/*
4555	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4556	* and serve to indicate exceptions to the rules.
4557	*/
4558
4559	/* Guest-natural selector base fields. */
4560	#if 0
4561	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4562	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4563	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4564	#endif
4565	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4566	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4567	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4568	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4569	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4570	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4571	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4572	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4573	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4574	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4575	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4576	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4577	#if 0
4578	/* Unused natural width guest-state fields. */
4579	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4580	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4581	#endif
4582	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4583	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4584
4585	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4586	#if 0
4587	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4588	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4589	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4590	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4591	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4592	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4593	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4594	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4595	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4596	#endif
4597
4598	/* Natural width guest-state fields. */
4599	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4600	#if 0
4601	/* Currently unused field. */
4602	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4603	#endif
4604
4605	if (pVM->hm.s.fNestedPaging)
4606	{
4607	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4608	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4609	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4610	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4611	}
4612	else
4613	{
4614	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4615	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4616	}
4617
4618	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4619	return VINF_SUCCESS;
4620	}
4621
4622
4623	/**
4624	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4625	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4626	* darwin, running 64-bit guests).
4627	*
4628	* @returns VBox status code.
4629	* @param pVCpu Pointer to the VMCPU.
4630	* @param idxField The VMCS field encoding.
4631	* @param u64Val 16, 32 or 64 bits value.
4632	*/
4633	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4634	{
4635	int rc;
4636	switch (idxField)
4637	{
4638	/*
4639	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4640	*/
4641	/* 64-bit Control fields. */
4642	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4643	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4644	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4645	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4646	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4647	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4648	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4649	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4650	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4651	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4652	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4653	case VMX_VMCS64_CTRL_EPTP_FULL:
4654	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4655	/* 64-bit Guest-state fields. */
4656	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4657	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4658	case VMX_VMCS64_GUEST_PAT_FULL:
4659	case VMX_VMCS64_GUEST_EFER_FULL:
4660	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4661	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4662	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4663	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4664	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4665	/* 64-bit Host-state fields. */
4666	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4667	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4668	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4669	{
4670	rc = VMXWriteVmcs32(idxField, u64Val);
4671	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4672	break;
4673	}
4674
4675	/*
4676	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4677	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4678	*/
4679	/* Natural-width Guest-state fields. */
4680	case VMX_VMCS_GUEST_CR3:
4681	case VMX_VMCS_GUEST_ES_BASE:
4682	case VMX_VMCS_GUEST_CS_BASE:
4683	case VMX_VMCS_GUEST_SS_BASE:
4684	case VMX_VMCS_GUEST_DS_BASE:
4685	case VMX_VMCS_GUEST_FS_BASE:
4686	case VMX_VMCS_GUEST_GS_BASE:
4687	case VMX_VMCS_GUEST_LDTR_BASE:
4688	case VMX_VMCS_GUEST_TR_BASE:
4689	case VMX_VMCS_GUEST_GDTR_BASE:
4690	case VMX_VMCS_GUEST_IDTR_BASE:
4691	case VMX_VMCS_GUEST_RSP:
4692	case VMX_VMCS_GUEST_RIP:
4693	case VMX_VMCS_GUEST_SYSENTER_ESP:
4694	case VMX_VMCS_GUEST_SYSENTER_EIP:
4695	{
4696	if (!(u64Val >> 32))
4697	{
4698	/* If this field is 64-bit, VT-x will zero out the top bits. */
4699	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4700	}
4701	else
4702	{
4703	/* Assert that only the 32->64 switcher case should ever come here. */
4704	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4705	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4706	}
4707	break;
4708	}
4709
4710	default:
4711	{
4712	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4713	rc = VERR_INVALID_PARAMETER;
4714	break;
4715	}
4716	}
4717	AssertRCReturn(rc, rc);
4718	return rc;
4719	}
4720
4721
4722	/**
4723	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4724	* hosts (except darwin) for 64-bit guests.
4725	*
4726	* @param pVCpu Pointer to the VMCPU.
4727	* @param idxField The VMCS field encoding.
4728	* @param u64Val 16, 32 or 64 bits value.
4729	*/
4730	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4731	{
4732	AssertPtr(pVCpu);
4733	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4734
4735	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4736	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4737
4738	/* Make sure there are no duplicates. */
4739	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4740	{
4741	if (pCache->Write.aField[i] == idxField)
4742	{
4743	pCache->Write.aFieldVal[i] = u64Val;
4744	return VINF_SUCCESS;
4745	}
4746	}
4747
4748	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4749	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4750	pCache->Write.cValidEntries++;
4751	return VINF_SUCCESS;
4752	}
4753
4754	/* Enable later when the assembly code uses these as callbacks. */
4755	#if 0
4756	/*
4757	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4758	*
4759	* @param pVCpu Pointer to the VMCPU.
4760	* @param pCache Pointer to the VMCS cache.
4761	*
4762	* @remarks No-long-jump zone!!!
4763	*/
4764	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4765	{
4766	AssertPtr(pCache);
4767	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4768	{
4769	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4770	AssertRC(rc);
4771	}
4772	pCache->Write.cValidEntries = 0;
4773	}
4774
4775
4776	/**
4777	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4778	*
4779	* @param pVCpu Pointer to the VMCPU.
4780	* @param pCache Pointer to the VMCS cache.
4781	*
4782	* @remarks No-long-jump zone!!!
4783	*/
4784	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4785	{
4786	AssertPtr(pCache);
4787	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4788	{
4789	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4790	AssertRC(rc);
4791	}
4792	}
4793	#endif
4794	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4795
4796
4797	/**
4798	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4799	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4800	* timer.
4801	*
4802	* @returns VBox status code.
4803	* @param pVCpu Pointer to the VMCPU.
4804	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805	* out-of-sync. Make sure to update the required fields
4806	* before using them.
4807	* @remarks No-long-jump zone!!!
4808	*/
4809	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4810	{
4811	int rc = VERR_INTERNAL_ERROR_5;
4812	bool fOffsettedTsc = false;
4813	PVM pVM = pVCpu->CTX_SUFF(pVM);
4814	if (pVM->hm.s.vmx.fUsePreemptTimer)
4815	{
4816	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4817
4818	/* Make sure the returned values have sane upper and lower boundaries. */
4819	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4820	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4821	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4822	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4823
4824	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4825	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4826	}
4827	else
4828	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4829
4830	if (fOffsettedTsc)
4831	{
4832	uint64_t u64CurTSC = ASMReadTSC();
4833	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4834	{
4835	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4836	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4837
4838	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4839	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4840	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4841	}
4842	else
4843	{
4844	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4845	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4846	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4847	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4848	}
4849	}
4850	else
4851	{
4852	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4853	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4854	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4855	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4856	}
4857	}
4858
4859
4860	/**
4861	* Determines if an exception is a contributory exception. Contributory
4862	* exceptions are ones which can cause double-faults. Page-fault is
4863	* intentionally not included here as it's a conditional contributory exception.
4864	*
4865	* @returns true if the exception is contributory, false otherwise.
4866	* @param uVector The exception vector.
4867	*/
4868	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4869	{
4870	switch (uVector)
4871	{
4872	case X86_XCPT_GP:
4873	case X86_XCPT_SS:
4874	case X86_XCPT_NP:
4875	case X86_XCPT_TS:
4876	case X86_XCPT_DE:
4877	return true;
4878	default:
4879	break;
4880	}
4881	return false;
4882	}
4883
4884
4885	/**
4886	* Sets an event as a pending event to be injected into the guest.
4887	*
4888	* @param pVCpu Pointer to the VMCPU.
4889	* @param u32IntrInfo The VM-entry interruption-information field.
4890	* @param cbInstr The VM-entry instruction length in bytes (for software
4891	* interrupts, exceptions and privileged software
4892	* exceptions).
4893	* @param u32ErrCode The VM-entry exception error code.
4894	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4895	* page-fault.
4896	*
4897	* @remarks Statistics counter assumes this is a guest event being injected or
4898	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4899	* always incremented.
4900	*/
4901	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4902	RTGCUINTPTR GCPtrFaultAddress)
4903	{
4904	Assert(!pVCpu->hm.s.Event.fPending);
4905	pVCpu->hm.s.Event.fPending = true;
4906	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4907	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4908	pVCpu->hm.s.Event.cbInstr = cbInstr;
4909	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4910
4911	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4912	}
4913
4914
4915	/**
4916	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4917	*
4918	* @param pVCpu Pointer to the VMCPU.
4919	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4920	* out-of-sync. Make sure to update the required fields
4921	* before using them.
4922	*/
4923	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4924	{
4925	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4926	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4927	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4928	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4929	}
4930
4931
4932	/**
4933	* Handle a condition that occurred while delivering an event through the guest
4934	* IDT.
4935	*
4936	* @returns VBox status code (informational error codes included).
4937	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4938	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4939	* continue execution of the guest which will delivery the #DF.
4940	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4941	*
4942	* @param pVCpu Pointer to the VMCPU.
4943	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4944	* out-of-sync. Make sure to update the required fields
4945	* before using them.
4946	* @param pVmxTransient Pointer to the VMX transient structure.
4947	*
4948	* @remarks No-long-jump zone!!!
4949	*/
4950	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4951	{
4952	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4953	AssertRCReturn(rc, rc);
4954	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4955	{
4956	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4957	AssertRCReturn(rc, rc);
4958
4959	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4960	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4961	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4962
4963	typedef enum
4964	{
4965	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4966	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4967	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4968	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4969	} VMXREFLECTXCPT;
4970
4971	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4972	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4973	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4974	{
4975	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4976	{
4977	enmReflect = VMXREFLECTXCPT_XCPT;
4978	#ifdef VBOX_STRICT
4979	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4980	&& uExitVector == X86_XCPT_PF)
4981	{
4982	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4983	}
4984	#endif
4985	if ( uExitVector == X86_XCPT_PF
4986	&& uIdtVector == X86_XCPT_PF)
4987	{
4988	pVmxTransient->fVectoringPF = true;
4989	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4990	}
4991	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4992	&& hmR0VmxIsContributoryXcpt(uExitVector)
4993	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4994	\|\| uIdtVector == X86_XCPT_PF))
4995	{
4996	enmReflect = VMXREFLECTXCPT_DF;
4997	}
4998	else if (uIdtVector == X86_XCPT_DF)
4999	enmReflect = VMXREFLECTXCPT_TF;
5000	}
5001	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5002	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5003	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5004	{
5005	/*
5006	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5007	* (whatever they are) as they reoccur when restarting the instruction.
5008	*/
5009	enmReflect = VMXREFLECTXCPT_XCPT;
5010	}
5011	}
5012	else
5013	{
5014	/*
5015	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5016	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5017	* original exception to the guest after handling the VM-exit.
5018	*/
5019	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5020	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5021	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5022	{
5023	enmReflect = VMXREFLECTXCPT_XCPT;
5024	}
5025	}
5026
5027	switch (enmReflect)
5028	{
5029	case VMXREFLECTXCPT_XCPT:
5030	{
5031	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5032	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5033	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5034
5035	uint32_t u32ErrCode = 0;
5036	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5037	{
5038	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5039	AssertRCReturn(rc, rc);
5040	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5041	}
5042
5043	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5044	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5045	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5046	rc = VINF_SUCCESS;
5047	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5048	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5049
5050	break;
5051	}
5052
5053	case VMXREFLECTXCPT_DF:
5054	{
5055	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5056	rc = VINF_HM_DOUBLE_FAULT;
5057	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5058	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5059
5060	break;
5061	}
5062
5063	case VMXREFLECTXCPT_TF:
5064	{
5065	rc = VINF_EM_RESET;
5066	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5067	uExitVector));
5068	break;
5069	}
5070
5071	default:
5072	Assert(rc == VINF_SUCCESS);
5073	break;
5074	}
5075	}
5076	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5077	return rc;
5078	}
5079
5080
5081	/**
5082	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5083	*
5084	* @returns VBox status code.
5085	* @param pVCpu Pointer to the VMCPU.
5086	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5087	* out-of-sync. Make sure to update the required fields
5088	* before using them.
5089	*
5090	* @remarks No-long-jump zone!!!
5091	*/
5092	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5093	{
5094	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5095	{
5096	uint32_t uVal = 0;
5097	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5098	AssertRCReturn(rc, rc);
5099	uint32_t uShadow = 0;
5100	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5101	AssertRCReturn(rc, rc);
5102
5103	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5104	CPUMSetGuestCR0(pVCpu, uVal);
5105	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5106	}
5107	return VINF_SUCCESS;
5108	}
5109
5110
5111	/**
5112	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5113	*
5114	* @returns VBox status code.
5115	* @param pVCpu Pointer to the VMCPU.
5116	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5117	* out-of-sync. Make sure to update the required fields
5118	* before using them.
5119	*
5120	* @remarks No-long-jump zone!!!
5121	*/
5122	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5123	{
5124	int rc = VINF_SUCCESS;
5125	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5126	{
5127	uint32_t uVal = 0;
5128	uint32_t uShadow = 0;
5129	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5130	AssertRCReturn(rc, rc);
5131	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5132	AssertRCReturn(rc, rc);
5133
5134	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5135	CPUMSetGuestCR4(pVCpu, uVal);
5136	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5137	}
5138	return rc;
5139	}
5140
5141
5142	/**
5143	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5144	*
5145	* @returns VBox status code.
5146	* @param pVCpu Pointer to the VMCPU.
5147	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5148	* out-of-sync. Make sure to update the required fields
5149	* before using them.
5150	*
5151	* @remarks No-long-jump zone!!!
5152	*/
5153	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5154	{
5155	int rc = VINF_SUCCESS;
5156	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5157	{
5158	uint64_t u64Val = 0;
5159	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5160	AssertRCReturn(rc, rc);
5161
5162	pMixedCtx->rip = u64Val;
5163	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5164	}
5165	return rc;
5166	}
5167
5168
5169	/**
5170	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5171	*
5172	* @returns VBox status code.
5173	* @param pVCpu Pointer to the VMCPU.
5174	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5175	* out-of-sync. Make sure to update the required fields
5176	* before using them.
5177	*
5178	* @remarks No-long-jump zone!!!
5179	*/
5180	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5181	{
5182	int rc = VINF_SUCCESS;
5183	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5184	{
5185	uint64_t u64Val = 0;
5186	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5187	AssertRCReturn(rc, rc);
5188
5189	pMixedCtx->rsp = u64Val;
5190	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5191	}
5192	return rc;
5193	}
5194
5195
5196	/**
5197	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5198	*
5199	* @returns VBox status code.
5200	* @param pVCpu Pointer to the VMCPU.
5201	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5202	* out-of-sync. Make sure to update the required fields
5203	* before using them.
5204	*
5205	* @remarks No-long-jump zone!!!
5206	*/
5207	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5208	{
5209	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5210	{
5211	uint32_t uVal = 0;
5212	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5213	AssertRCReturn(rc, rc);
5214
5215	pMixedCtx->eflags.u32 = uVal;
5216	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5217	{
5218	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5219	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5220
5221	pMixedCtx->eflags.Bits.u1VM = 0;
5222	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5223	}
5224
5225	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5226	}
5227	return VINF_SUCCESS;
5228	}
5229
5230
5231	/**
5232	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5233	* guest-CPU context.
5234	*/
5235	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5236	{
5237	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5238	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5239	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5240	return rc;
5241	}
5242
5243
5244	/**
5245	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5246	* from the guest-state area in the VMCS.
5247	*
5248	* @param pVCpu Pointer to the VMCPU.
5249	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5250	* out-of-sync. Make sure to update the required fields
5251	* before using them.
5252	*
5253	* @remarks No-long-jump zone!!!
5254	*/
5255	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5256	{
5257	uint32_t uIntrState = 0;
5258	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5259	AssertRC(rc);
5260
5261	if (!uIntrState)
5262	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5263	else
5264	{
5265	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5266	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5267	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5268	AssertRC(rc);
5269	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5270	AssertRC(rc);
5271
5272	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5273	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5274	}
5275	}
5276
5277
5278	/**
5279	* Saves the guest's activity state.
5280	*
5281	* @returns VBox status code.
5282	* @param pVCpu Pointer to the VMCPU.
5283	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5284	* out-of-sync. Make sure to update the required fields
5285	* before using them.
5286	*
5287	* @remarks No-long-jump zone!!!
5288	*/
5289	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5290	{
5291	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5292	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5293	return VINF_SUCCESS;
5294	}
5295
5296
5297	/**
5298	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5299	* the current VMCS into the guest-CPU context.
5300	*
5301	* @returns VBox status code.
5302	* @param pVCpu Pointer to the VMCPU.
5303	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5304	* out-of-sync. Make sure to update the required fields
5305	* before using them.
5306	*
5307	* @remarks No-long-jump zone!!!
5308	*/
5309	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5310	{
5311	int rc = VINF_SUCCESS;
5312	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5313	{
5314	uint32_t u32Val = 0;
5315	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5316	pMixedCtx->SysEnter.cs = u32Val;
5317	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5318	}
5319
5320	uint64_t u64Val = 0;
5321	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5322	{
5323	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5324	pMixedCtx->SysEnter.eip = u64Val;
5325	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5326	}
5327	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5328	{
5329	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5330	pMixedCtx->SysEnter.esp = u64Val;
5331	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5332	}
5333	return rc;
5334	}
5335
5336
5337	/**
5338	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5339	* context.
5340	*
5341	* @returns VBox status code.
5342	* @param pVCpu Pointer to the VMCPU.
5343	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5344	* out-of-sync. Make sure to update the required fields
5345	* before using them.
5346	*
5347	* @remarks No-long-jump zone!!!
5348	*/
5349	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5350	{
5351	int rc = VINF_SUCCESS;
5352	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5353	{
5354	uint64_t u64Val = 0;
5355	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5356	pMixedCtx->fs.u64Base = u64Val;
5357	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5358	}
5359	return rc;
5360	}
5361
5362
5363	/**
5364	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5365	* context.
5366	*
5367	* @returns VBox status code.
5368	* @param pVCpu Pointer to the VMCPU.
5369	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5370	* out-of-sync. Make sure to update the required fields
5371	* before using them.
5372	*
5373	* @remarks No-long-jump zone!!!
5374	*/
5375	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5376	{
5377	int rc = VINF_SUCCESS;
5378	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5379	{
5380	uint64_t u64Val = 0;
5381	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5382	pMixedCtx->gs.u64Base = u64Val;
5383	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5384	}
5385	return rc;
5386	}
5387
5388
5389	/**
5390	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5391	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5392	* and TSC_AUX.
5393	*
5394	* @returns VBox status code.
5395	* @param pVCpu Pointer to the VMCPU.
5396	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5397	* out-of-sync. Make sure to update the required fields
5398	* before using them.
5399	*
5400	* @remarks No-long-jump zone!!!
5401	*/
5402	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5403	{
5404	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5405	return VINF_SUCCESS;
5406
5407	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5408	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5409	{
5410	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5411	pMsr += i;
5412	switch (pMsr->u32Msr)
5413	{
5414	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5415	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5416	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5417	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5418	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5419	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5420	default:
5421	{
5422	AssertFailed();
5423	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5424	}
5425	}
5426	}
5427	#endif
5428
5429	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5430	return VINF_SUCCESS;
5431	}
5432
5433
5434	/**
5435	* Saves the guest control registers from the current VMCS into the guest-CPU
5436	* context.
5437	*
5438	* @returns VBox status code.
5439	* @param pVCpu Pointer to the VMCPU.
5440	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5441	* out-of-sync. Make sure to update the required fields
5442	* before using them.
5443	*
5444	* @remarks No-long-jump zone!!!
5445	*/
5446	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5447	{
5448	/* Guest CR0. Guest FPU. */
5449	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5450	AssertRCReturn(rc, rc);
5451
5452	/* Guest CR4. */
5453	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5454	AssertRCReturn(rc, rc);
5455
5456	/* Guest CR2 - updated always during the world-switch or in #PF. */
5457	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5458	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5459	{
5460	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5461	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5462
5463	PVM pVM = pVCpu->CTX_SUFF(pVM);
5464	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5465	\|\| ( pVM->hm.s.fNestedPaging
5466	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5467	{
5468	uint64_t u64Val = 0;
5469	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5470	if (pMixedCtx->cr3 != u64Val)
5471	{
5472	CPUMSetGuestCR3(pVCpu, u64Val);
5473	if (VMMRZCallRing3IsEnabled(pVCpu))
5474	{
5475	PGMUpdateCR3(pVCpu, u64Val);
5476	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5477	}
5478	else
5479	{
5480	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5481	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5482	}
5483	}
5484
5485	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5486	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5487	{
5488	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5489	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5490	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5491	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5492
5493	if (VMMRZCallRing3IsEnabled(pVCpu))
5494	{
5495	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5496	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5497	}
5498	else
5499	{
5500	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5501	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5502	}
5503	}
5504	}
5505
5506	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5507	}
5508
5509	/*
5510	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5511	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5512	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5513	*
5514	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5515	*/
5516	if (VMMRZCallRing3IsEnabled(pVCpu))
5517	{
5518	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5519	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5520
5521	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5522	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5523
5524	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5525	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5526	}
5527
5528	return rc;
5529	}
5530
5531
5532	/**
5533	* Reads a guest segment register from the current VMCS into the guest-CPU
5534	* context.
5535	*
5536	* @returns VBox status code.
5537	* @param pVCpu Pointer to the VMCPU.
5538	* @param idxSel Index of the selector in the VMCS.
5539	* @param idxLimit Index of the segment limit in the VMCS.
5540	* @param idxBase Index of the segment base in the VMCS.
5541	* @param idxAccess Index of the access rights of the segment in the VMCS.
5542	* @param pSelReg Pointer to the segment selector.
5543	*
5544	* @remarks No-long-jump zone!!!
5545	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5546	* macro as that takes care of whether to read from the VMCS cache or
5547	* not.
5548	*/
5549	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5550	PCPUMSELREG pSelReg)
5551	{
5552	uint32_t u32Val = 0;
5553	int rc = VMXReadVmcs32(idxSel, &u32Val);
5554	AssertRCReturn(rc, rc);
5555	pSelReg->Sel = (uint16_t)u32Val;
5556	pSelReg->ValidSel = (uint16_t)u32Val;
5557	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5558
5559	rc = VMXReadVmcs32(idxLimit, &u32Val);
5560	AssertRCReturn(rc, rc);
5561	pSelReg->u32Limit = u32Val;
5562
5563	uint64_t u64Val = 0;
5564	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5565	AssertRCReturn(rc, rc);
5566	pSelReg->u64Base = u64Val;
5567
5568	rc = VMXReadVmcs32(idxAccess, &u32Val);
5569	AssertRCReturn(rc, rc);
5570	pSelReg->Attr.u = u32Val;
5571
5572	/*
5573	* If VT-x marks the segment as unusable, most other bits remain undefined:
5574	* - For CS the L, D and G bits have meaning.
5575	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5576	* - For the remaining data segments no bits are defined.
5577	*
5578	* The present bit and the unusable bit has been observed to be set at the
5579	* same time (the selector was supposed to invalid as we started executing
5580	* a V8086 interrupt in ring-0).
5581	*
5582	* What should be important for the rest of the VBox code that the P bit is
5583	* cleared. Some of the other VBox code recognizes the unusable bit, but
5584	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5585	* safe side here, we'll strip off P and other bits we don't care about. If
5586	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5587	*
5588	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5589	*/
5590	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5591	{
5592	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5593
5594	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5595	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5596	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5597
5598	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5599	#ifdef DEBUG_bird
5600	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5601	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5602	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5603	#endif
5604	}
5605	return VINF_SUCCESS;
5606	}
5607
5608
5609	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5610	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5611	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5612	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5613	#else
5614	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5615	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5616	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5617	#endif
5618
5619
5620	/**
5621	* Saves the guest segment registers from the current VMCS into the guest-CPU
5622	* context.
5623	*
5624	* @returns VBox status code.
5625	* @param pVCpu Pointer to the VMCPU.
5626	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5627	* out-of-sync. Make sure to update the required fields
5628	* before using them.
5629	*
5630	* @remarks No-long-jump zone!!!
5631	*/
5632	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5633	{
5634	/* Guest segment registers. */
5635	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5636	{
5637	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5638	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5639	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5640	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5641	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5642	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5643	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5644
5645	/* Restore segment attributes for real-on-v86 mode hack. */
5646	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5647	{
5648	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5649	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5650	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5651	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5652	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5653	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5654	}
5655	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5656	}
5657
5658	return VINF_SUCCESS;
5659	}
5660
5661
5662	/**
5663	* Saves the guest descriptor table registers and task register from the current
5664	* VMCS into the guest-CPU context.
5665	*
5666	* @returns VBox status code.
5667	* @param pVCpu Pointer to the VMCPU.
5668	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5669	* out-of-sync. Make sure to update the required fields
5670	* before using them.
5671	*
5672	* @remarks No-long-jump zone!!!
5673	*/
5674	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5675	{
5676	int rc = VINF_SUCCESS;
5677
5678	/* Guest LDTR. */
5679	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5680	{
5681	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5682	AssertRCReturn(rc, rc);
5683	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5684	}
5685
5686	/* Guest GDTR. */
5687	uint64_t u64Val = 0;
5688	uint32_t u32Val = 0;
5689	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5690	{
5691	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5692	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5693	pMixedCtx->gdtr.pGdt = u64Val;
5694	pMixedCtx->gdtr.cbGdt = u32Val;
5695	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5696	}
5697
5698	/* Guest IDTR. */
5699	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5700	{
5701	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5702	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5703	pMixedCtx->idtr.pIdt = u64Val;
5704	pMixedCtx->idtr.cbIdt = u32Val;
5705	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5706	}
5707
5708	/* Guest TR. */
5709	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5710	{
5711	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5712	AssertRCReturn(rc, rc);
5713
5714	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5715	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5716	{
5717	rc = VMXLOCAL_READ_SEG(TR, tr);
5718	AssertRCReturn(rc, rc);
5719	}
5720	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5721	}
5722	return rc;
5723	}
5724
5725	#undef VMXLOCAL_READ_SEG
5726
5727
5728	/**
5729	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5730	* context.
5731	*
5732	* @returns VBox status code.
5733	* @param pVCpu Pointer to the VMCPU.
5734	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5735	* out-of-sync. Make sure to update the required fields
5736	* before using them.
5737	*
5738	* @remarks No-long-jump zone!!!
5739	*/
5740	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5741	{
5742	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5743	{
5744	if (!CPUMIsHyperDebugStateActive(pVCpu))
5745	{
5746	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5747	uint32_t u32Val;
5748	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5749	pMixedCtx->dr[7] = u32Val;
5750	}
5751
5752	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5753	}
5754	return VINF_SUCCESS;
5755	}
5756
5757
5758	/**
5759	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5760	*
5761	* @returns VBox status code.
5762	* @param pVCpu Pointer to the VMCPU.
5763	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5764	* out-of-sync. Make sure to update the required fields
5765	* before using them.
5766	*
5767	* @remarks No-long-jump zone!!!
5768	*/
5769	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5770	{
5771	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5772	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5773	return VINF_SUCCESS;
5774	}
5775
5776
5777	/**
5778	* Saves the entire guest state from the currently active VMCS into the
5779	* guest-CPU context. This essentially VMREADs all guest-data.
5780	*
5781	* @returns VBox status code.
5782	* @param pVCpu Pointer to the VMCPU.
5783	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5784	* out-of-sync. Make sure to update the required fields
5785	* before using them.
5786	*/
5787	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5788	{
5789	Assert(pVCpu);
5790	Assert(pMixedCtx);
5791
5792	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5793	return VINF_SUCCESS;
5794
5795	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5796	again on the ring-3 callback path, there is no real need to. */
5797	if (VMMRZCallRing3IsEnabled(pVCpu))
5798	VMMR0LogFlushDisable(pVCpu);
5799	else
5800	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5801	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5802
5803	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5804	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5805
5806	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5807	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5808
5809	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5810	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5811
5812	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5813	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5814
5815	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5816	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5817
5818	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5819	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5820
5821	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5822	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5823
5824	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5825	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5826
5827	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5828	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5829
5830	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5831	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5832
5833	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5834	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5835
5836	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5837	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5838
5839	if (VMMRZCallRing3IsEnabled(pVCpu))
5840	VMMR0LogFlushEnable(pVCpu);
5841
5842	return rc;
5843	}
5844
5845
5846	/**
5847	* Check per-VM and per-VCPU force flag actions that require us to go back to
5848	* ring-3 for one reason or another.
5849	*
5850	* @returns VBox status code (information status code included).
5851	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5852	* ring-3.
5853	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5854	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5855	* interrupts)
5856	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5857	* all EMTs to be in ring-3.
5858	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5859	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5860	* to the EM loop.
5861	*
5862	* @param pVM Pointer to the VM.
5863	* @param pVCpu Pointer to the VMCPU.
5864	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5865	* out-of-sync. Make sure to update the required fields
5866	* before using them.
5867	*/
5868	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5869	{
5870	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5871
5872	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5873	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5874	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5875	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5876	{
5877	/* We need the control registers now, make sure the guest-CPU context is updated. */
5878	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5879	AssertRCReturn(rc3, rc3);
5880
5881	/* Pending HM CR3 sync. */
5882	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5883	{
5884	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5885	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
5886	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5887	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5888	}
5889
5890	/* Pending HM PAE PDPEs. */
5891	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5892	{
5893	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5894	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5895	}
5896
5897	/* Pending PGM C3 sync. */
5898	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5899	{
5900	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
5901	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5902	if (rc2 != VINF_SUCCESS)
5903	{
5904	AssertRC(rc2);
5905	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5906	return rc2;
5907	}
5908	}
5909
5910	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5911	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5912	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5913	{
5914	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5915	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5916	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5917	return rc2;
5918	}
5919
5920	/* Pending VM request packets, such as hardware interrupts. */
5921	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5922	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5923	{
5924	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5925	return VINF_EM_PENDING_REQUEST;
5926	}
5927
5928	/* Pending PGM pool flushes. */
5929	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5930	{
5931	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5932	return VINF_PGM_POOL_FLUSH_PENDING;
5933	}
5934
5935	/* Pending DMA requests. */
5936	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5937	{
5938	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5939	return VINF_EM_RAW_TO_R3;
5940	}
5941	}
5942
5943	return VINF_SUCCESS;
5944	}
5945
5946
5947	/**
5948	* Converts any TRPM trap into a pending HM event. This is typically used when
5949	* entering from ring-3 (not longjmp returns).
5950	*
5951	* @param pVCpu Pointer to the VMCPU.
5952	*/
5953	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5954	{
5955	Assert(TRPMHasTrap(pVCpu));
5956	Assert(!pVCpu->hm.s.Event.fPending);
5957
5958	uint8_t uVector;
5959	TRPMEVENT enmTrpmEvent;
5960	RTGCUINT uErrCode;
5961	RTGCUINTPTR GCPtrFaultAddress;
5962	uint8_t cbInstr;
5963
5964	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5965	AssertRC(rc);
5966
5967	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5968	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5969	if (enmTrpmEvent == TRPM_TRAP)
5970	{
5971	switch (uVector)
5972	{
5973	case X86_XCPT_BP:
5974	case X86_XCPT_OF:
5975	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5976	break;
5977
5978	case X86_XCPT_PF:
5979	case X86_XCPT_DF:
5980	case X86_XCPT_TS:
5981	case X86_XCPT_NP:
5982	case X86_XCPT_SS:
5983	case X86_XCPT_GP:
5984	case X86_XCPT_AC:
5985	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5986	/* no break! */
5987	default:
5988	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5989	break;
5990	}
5991	}
5992	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5993	{
5994	if (uVector == X86_XCPT_NMI)
5995	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5996	else
5997	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5998	}
5999	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6000	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6001	else
6002	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6003
6004	rc = TRPMResetTrap(pVCpu);
6005	AssertRC(rc);
6006	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6007	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6008
6009	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6010	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6011	}
6012
6013
6014	/**
6015	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6016	* VT-x to execute any instruction.
6017	*
6018	* @param pvCpu Pointer to the VMCPU.
6019	*/
6020	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6021	{
6022	Assert(pVCpu->hm.s.Event.fPending);
6023
6024	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6025	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6026	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6027	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6028
6029	/* If a trap was already pending, we did something wrong! */
6030	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6031
6032	TRPMEVENT enmTrapType;
6033	switch (uVectorType)
6034	{
6035	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6036	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6037	enmTrapType = TRPM_HARDWARE_INT;
6038	break;
6039
6040	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6041	enmTrapType = TRPM_SOFTWARE_INT;
6042	break;
6043
6044	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6045	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6046	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6047	enmTrapType = TRPM_TRAP;
6048	break;
6049
6050	default:
6051	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6052	enmTrapType = TRPM_32BIT_HACK;
6053	break;
6054	}
6055
6056	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6057
6058	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6059	AssertRC(rc);
6060
6061	if (fErrorCodeValid)
6062	TRPMSetErrorCode(pVCpu, uErrorCode);
6063
6064	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6065	&& uVector == X86_XCPT_PF)
6066	{
6067	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6068	}
6069	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6070	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6071	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6072	{
6073	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6074	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6075	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6076	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6077	}
6078	pVCpu->hm.s.Event.fPending = false;
6079	}
6080
6081
6082	/**
6083	* Does the necessary state syncing before returning to ring-3 for any reason
6084	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6085	*
6086	* @returns VBox status code.
6087	* @param pVM Pointer to the VM.
6088	* @param pVCpu Pointer to the VMCPU.
6089	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6090	* out-of-sync. Make sure to update the required fields
6091	* before using them.
6092	*
6093	* @remarks No-long-jmp zone!!!
6094	*/
6095	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6096	{
6097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6098	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6099
6100	RTCPUID idCpu = RTMpCpuId();
6101	Log4Func(("HostCpuId=%u\n", idCpu));
6102
6103	/* Save the guest state if necessary. */
6104	if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6105	{
6106	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6107	AssertRCReturn(rc, rc);
6108	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6109	}
6110
6111	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6112	if (CPUMIsGuestFPUStateActive(pVCpu))
6113	{
6114	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6115	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6116	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
6117	}
6118
6119	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6120	#ifdef VBOX_STRICT
6121	if (CPUMIsHyperDebugStateActive(pVCpu))
6122	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6123	#endif
6124	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6125	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
6126	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6127	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6128
6129	/* Restore host-state bits that VT-x only restores partially. */
6130	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6131	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6132	{
6133	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6134	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6135	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6136	}
6137
6138	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6139	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6140	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6141	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6142	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6143	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6144	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6145	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6146
6147	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6148
6149	/** @todo This kinda defeats the purpose of having preemption hooks.
6150	* The problem is, deregistering the hooks should be moved to a place that
6151	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6152	* context.
6153	*/
6154	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6155	{
6156	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6157	AssertRCReturn(rc, rc);
6158
6159	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6160	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6161	}
6162	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6163	NOREF(idCpu);
6164
6165	return VINF_SUCCESS;
6166	}
6167
6168
6169	/**
6170	* Leaves the VT-x session.
6171	*
6172	* @returns VBox status code.
6173	* @param pVM Pointer to the VM.
6174	* @param pVCpu Pointer to the VMCPU.
6175	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6176	* out-of-sync. Make sure to update the required fields
6177	* before using them.
6178	*
6179	* @remarks No-long-jmp zone!!!
6180	*/
6181	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6182	{
6183	HM_DISABLE_PREEMPT_IF_NEEDED();
6184	HMVMX_ASSERT_CPU_SAFE();
6185	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6186	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6187
6188	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6189	and done this from the VMXR0ThreadCtxCallback(). */
6190	if (!pVCpu->hm.s.fLeaveDone)
6191	{
6192	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6193	AssertRCReturn(rc2, rc2);
6194	pVCpu->hm.s.fLeaveDone = true;
6195	}
6196
6197	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6198	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6199	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6200	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6201	VMMR0ThreadCtxHooksDeregister(pVCpu);
6202
6203	/* Leave HM context. This takes care of local init (term). */
6204	int rc = HMR0LeaveCpu(pVCpu);
6205
6206	HM_RESTORE_PREEMPT_IF_NEEDED();
6207
6208	return rc;
6209	}
6210
6211
6212	/**
6213	* Does the necessary state syncing before doing a longjmp to ring-3.
6214	*
6215	* @returns VBox status code.
6216	* @param pVM Pointer to the VM.
6217	* @param pVCpu Pointer to the VMCPU.
6218	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6219	* out-of-sync. Make sure to update the required fields
6220	* before using them.
6221	*
6222	* @remarks No-long-jmp zone!!!
6223	*/
6224	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6225	{
6226	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6227	}
6228
6229
6230	/**
6231	* Take necessary actions before going back to ring-3.
6232	*
6233	* An action requires us to go back to ring-3. This function does the necessary
6234	* steps before we can safely return to ring-3. This is not the same as longjmps
6235	* to ring-3, this is voluntary and prepares the guest so it may continue
6236	* executing outside HM (recompiler/IEM).
6237	*
6238	* @returns VBox status code.
6239	* @param pVM Pointer to the VM.
6240	* @param pVCpu Pointer to the VMCPU.
6241	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6242	* out-of-sync. Make sure to update the required fields
6243	* before using them.
6244	* @param rcExit The reason for exiting to ring-3. Can be
6245	* VINF_VMM_UNKNOWN_RING3_CALL.
6246	*/
6247	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6248	{
6249	Assert(pVM);
6250	Assert(pVCpu);
6251	Assert(pMixedCtx);
6252	HMVMX_ASSERT_PREEMPT_SAFE();
6253
6254	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6255	{
6256	/* We've done what is required in hmR0VmxExitErrInvalidGuestState(). We're not going to continue guest execution... */
6257	return VINF_SUCCESS;
6258	}
6259	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6260	{
6261	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6262	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6263	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6264	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6265	return VINF_SUCCESS;
6266	}
6267
6268	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6269	VMMRZCallRing3Disable(pVCpu);
6270	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6271
6272	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6273	if (pVCpu->hm.s.Event.fPending)
6274	{
6275	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6276	Assert(!pVCpu->hm.s.Event.fPending);
6277	}
6278
6279	/* Save guest state and restore host state bits. */
6280	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6281	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6282
6283	/* Sync recompiler state. */
6284	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6285	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6286	\| CPUM_CHANGED_LDTR
6287	\| CPUM_CHANGED_GDTR
6288	\| CPUM_CHANGED_IDTR
6289	\| CPUM_CHANGED_TR
6290	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6291	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6292	if ( pVM->hm.s.fNestedPaging
6293	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6294	{
6295	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6296	}
6297
6298	/*
6299	* Clear the X86_EFL_TF if necessary.
6300	*/
6301	if (pVCpu->hm.s.fClearTrapFlag)
6302	{
6303	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6304	pMixedCtx->eflags.Bits.u1TF = 0;
6305	pVCpu->hm.s.fClearTrapFlag = false;
6306	}
6307	/** @todo there seems to be issues with the resume flag when the monitor trap
6308	* flag is pending without being used. Seen early in bios init when
6309	* accessing APIC page in prot mode. */
6310
6311	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6312	if (rcExit != VINF_EM_RAW_INTERRUPT)
6313	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6314
6315	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6316
6317	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6318	VMMRZCallRing3RemoveNotification(pVCpu);
6319	VMMRZCallRing3Enable(pVCpu);
6320
6321	return rc;
6322	}
6323
6324
6325	/**
6326	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6327	* longjump to ring-3 and possibly get preempted.
6328	*
6329	* @returns VBox status code.
6330	* @param pVCpu Pointer to the VMCPU.
6331	* @param enmOperation The operation causing the ring-3 longjump.
6332	* @param pvUser Opaque pointer to the guest-CPU context. The data
6333	* may be out-of-sync. Make sure to update the required
6334	* fields before using them.
6335	*
6336	* @remarks Must never be called with @a enmOperation ==
6337	* VMMCALLRING3_VM_R0_ASSERTION. We can't assert it here because if it
6338	* it -does- get called with VMMCALLRING3_VM_R0_ASSERTION, we'll end up
6339	* with an infinite recursion.
6340	*/
6341	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6342	{
6343	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6344	Assert(pVCpu);
6345	Assert(pvUser);
6346	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6347	HMVMX_ASSERT_PREEMPT_SAFE();
6348
6349	VMMRZCallRing3Disable(pVCpu);
6350	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6351
6352	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6353	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6354	AssertRCReturn(rc, rc);
6355
6356	VMMRZCallRing3Enable(pVCpu);
6357	return VINF_SUCCESS;
6358	}
6359
6360
6361	/**
6362	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6363	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6364	*
6365	* @param pVCpu Pointer to the VMCPU.
6366	*/
6367	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6368	{
6369	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6370	{
6371	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6372	{
6373	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6374	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6375	AssertRC(rc);
6376	}
6377	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6378	}
6379
6380
6381	/**
6382	* Evaluates the event to be delivered to the guest and sets it as the pending
6383	* event.
6384	*
6385	* @param pVCpu Pointer to the VMCPU.
6386	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6387	* out-of-sync. Make sure to update the required fields
6388	* before using them.
6389	*/
6390	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6391	{
6392	Assert(!pVCpu->hm.s.Event.fPending);
6393
6394	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6395	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6396	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6397	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6398
6399	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6400	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6401	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6402	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6403	Assert(!TRPMHasTrap(pVCpu));
6404
6405	/** @todo SMI. SMIs take priority over NMIs. */
6406	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6407	{
6408	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6409	if ( !fBlockMovSS
6410	&& !fBlockSti)
6411	{
6412	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6413	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6414	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6415	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6416
6417	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6418	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6419	}
6420	else
6421	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6422	}
6423	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6424	&& !pVCpu->hm.s.fSingleInstruction)
6425	{
6426	/*
6427	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6428	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6429	* evaluated here and not set as pending, solely based on the force-flags.
6430	*/
6431	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6432	AssertRC(rc);
6433	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6434	if ( !fBlockInt
6435	&& !fBlockSti
6436	&& !fBlockMovSS)
6437	{
6438	uint8_t u8Interrupt;
6439	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6440	if (RT_SUCCESS(rc))
6441	{
6442	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6443	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6444	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6445
6446	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6447	}
6448	else
6449	{
6450	/** @todo Does this actually happen? If not turn it into an assertion. */
6451	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6452	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6453	}
6454	}
6455	else
6456	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6457	}
6458	}
6459
6460
6461	/**
6462	* Injects any pending events into the guest if the guest is in a state to
6463	* receive them.
6464	*
6465	* @returns VBox status code (informational status codes included).
6466	* @param pVCpu Pointer to the VMCPU.
6467	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6468	* out-of-sync. Make sure to update the required fields
6469	* before using them.
6470	*/
6471	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6472	{
6473	HMVMX_ASSERT_PREEMPT_SAFE();
6474	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6475
6476	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6477	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6478	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6479	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6480
6481	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6482	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6483	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6484	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6485	Assert(!TRPMHasTrap(pVCpu));
6486
6487	int rc = VINF_SUCCESS;
6488	if (pVCpu->hm.s.Event.fPending)
6489	{
6490	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
6491	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6492	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6493	{
6494	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6495	AssertRCReturn(rc, rc);
6496	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6497	Assert(!fBlockInt);
6498	Assert(!fBlockSti);
6499	Assert(!fBlockMovSS);
6500	}
6501	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6502	{
6503	Assert(!fBlockSti);
6504	Assert(!fBlockMovSS);
6505	}
6506	#endif
6507	Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6508	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6509	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6510	AssertRCReturn(rc, rc);
6511
6512	/* Update the interruptibility-state as it could have been changed by
6513	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6514	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6515	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6516
6517	#ifdef VBOX_WITH_STATISTICS
6518	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6519	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6520	else
6521	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6522	#endif
6523	}
6524
6525	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6526	int rc2 = VINF_SUCCESS;
6527	if ( fBlockSti
6528	\|\| fBlockMovSS)
6529	{
6530	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6531	{
6532	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6533	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6534	{
6535	/*
6536	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6537	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6538	* See Intel spec. 27.3.4 "Saving Non-Register State".
6539	*/
6540	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6541	AssertRCReturn(rc, rc);
6542	}
6543	}
6544	else
6545	{
6546	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6547	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6548	uIntrState = 0;
6549	}
6550	}
6551
6552	/*
6553	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6554	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6555	*/
6556	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6557	AssertRC(rc2);
6558
6559	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6560	return rc;
6561	}
6562
6563
6564	/**
6565	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6566	*
6567	* @param pVCpu Pointer to the VMCPU.
6568	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6569	* out-of-sync. Make sure to update the required fields
6570	* before using them.
6571	*/
6572	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6573	{
6574	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6575	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6576	}
6577
6578
6579	/**
6580	* Injects a double-fault (#DF) exception into the VM.
6581	*
6582	* @returns VBox status code (informational status code included).
6583	* @param pVCpu Pointer to the VMCPU.
6584	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6585	* out-of-sync. Make sure to update the required fields
6586	* before using them.
6587	*/
6588	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6589	{
6590	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6591	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6592	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6593	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6594	puIntrState);
6595	}
6596
6597
6598	/**
6599	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6600	*
6601	* @param pVCpu Pointer to the VMCPU.
6602	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6603	* out-of-sync. Make sure to update the required fields
6604	* before using them.
6605	*/
6606	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6607	{
6608	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6609	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6610	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6611	}
6612
6613
6614	/**
6615	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6616	*
6617	* @param pVCpu Pointer to the VMCPU.
6618	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6619	* out-of-sync. Make sure to update the required fields
6620	* before using them.
6621	* @param cbInstr The value of RIP that is to be pushed on the guest
6622	* stack.
6623	*/
6624	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6625	{
6626	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6627	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6628	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6629	}
6630
6631
6632	/**
6633	* Injects a general-protection (#GP) fault into the VM.
6634	*
6635	* @returns VBox status code (informational status code included).
6636	* @param pVCpu Pointer to the VMCPU.
6637	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6638	* out-of-sync. Make sure to update the required fields
6639	* before using them.
6640	* @param u32ErrorCode The error code associated with the #GP.
6641	*/
6642	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6643	uint32_t *puIntrState)
6644	{
6645	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6646	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6647	if (fErrorCodeValid)
6648	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6649	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6650	puIntrState);
6651	}
6652
6653
6654	/**
6655	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6656	*
6657	* @param pVCpu Pointer to the VMCPU.
6658	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6659	* out-of-sync. Make sure to update the required fields
6660	* before using them.
6661	* @param uVector The software interrupt vector number.
6662	* @param cbInstr The value of RIP that is to be pushed on the guest
6663	* stack.
6664	*/
6665	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6666	{
6667	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6668	if ( uVector == X86_XCPT_BP
6669	\|\| uVector == X86_XCPT_OF)
6670	{
6671	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6672	}
6673	else
6674	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6675	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6676	}
6677
6678
6679	/**
6680	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6681	* stack.
6682	*
6683	* @returns VBox status code (information status code included).
6684	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6685	* @param pVM Pointer to the VM.
6686	* @param pMixedCtx Pointer to the guest-CPU context.
6687	* @param uValue The value to push to the guest stack.
6688	*/
6689	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6690	{
6691	/*
6692	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6693	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6694	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6695	*/
6696	if (pMixedCtx->sp == 1)
6697	return VINF_EM_RESET;
6698	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6699	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6700	AssertRCReturn(rc, rc);
6701	return rc;
6702	}
6703
6704
6705	/**
6706	* Injects an event into the guest upon VM-entry by updating the relevant fields
6707	* in the VM-entry area in the VMCS.
6708	*
6709	* @returns VBox status code (informational error codes included).
6710	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6711	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6712	*
6713	* @param pVCpu Pointer to the VMCPU.
6714	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6715	* be out-of-sync. Make sure to update the required
6716	* fields before using them.
6717	* @param u64IntrInfo The VM-entry interruption-information field.
6718	* @param cbInstr The VM-entry instruction length in bytes (for
6719	* software interrupts, exceptions and privileged
6720	* software exceptions).
6721	* @param u32ErrCode The VM-entry exception error code.
6722	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6723	* @param puIntrState Pointer to the current guest interruptibility-state.
6724	* This interruptibility-state will be updated if
6725	* necessary. This cannot not be NULL.
6726	*
6727	* @remarks Requires CR0!
6728	* @remarks No-long-jump zone!!!
6729	*/
6730	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6731	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6732	{
6733	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6734	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6735	Assert(puIntrState);
6736	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6737
6738	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6739	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6740
6741	#ifdef VBOX_STRICT
6742	/* Validate the error-code-valid bit for hardware exceptions. */
6743	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6744	{
6745	switch (uVector)
6746	{
6747	case X86_XCPT_PF:
6748	case X86_XCPT_DF:
6749	case X86_XCPT_TS:
6750	case X86_XCPT_NP:
6751	case X86_XCPT_SS:
6752	case X86_XCPT_GP:
6753	case X86_XCPT_AC:
6754	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6755	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6756	/* fallthru */
6757	default:
6758	break;
6759	}
6760	}
6761	#endif
6762
6763	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6764	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6765	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6766
6767	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6768
6769	/* We require CR0 to check if the guest is in real-mode. */
6770	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6771	AssertRCReturn(rc, rc);
6772
6773	/*
6774	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6775	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6776	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6777	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6778	*/
6779	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6780	{
6781	PVM pVM = pVCpu->CTX_SUFF(pVM);
6782	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6783	{
6784	Assert(PDMVmmDevHeapIsEnabled(pVM));
6785	Assert(pVM->hm.s.vmx.pRealModeTSS);
6786
6787	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6788	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6789	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6790	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6791	AssertRCReturn(rc, rc);
6792	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6793
6794	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6795	const size_t cbIdtEntry = sizeof(X86IDTR16);
6796	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6797	{
6798	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6799	if (uVector == X86_XCPT_DF)
6800	return VINF_EM_RESET;
6801	else if (uVector == X86_XCPT_GP)
6802	{
6803	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6804	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6805	}
6806
6807	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6808	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6809	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6810	}
6811
6812	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6813	uint16_t uGuestIp = pMixedCtx->ip;
6814	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6815	{
6816	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6817	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6818	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6819	}
6820	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6821	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6822
6823	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6824	X86IDTR16 IdtEntry;
6825	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6826	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6827	AssertRCReturn(rc, rc);
6828
6829	/* Construct the stack frame for the interrupt/exception handler. */
6830	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6831	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6832	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6833	AssertRCReturn(rc, rc);
6834
6835	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6836	if (rc == VINF_SUCCESS)
6837	{
6838	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6839	pMixedCtx->rip = IdtEntry.offSel;
6840	pMixedCtx->cs.Sel = IdtEntry.uSel;
6841	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6842	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6843	&& uVector == X86_XCPT_PF)
6844	{
6845	pMixedCtx->cr2 = GCPtrFaultAddress;
6846	}
6847
6848	/* If any other guest-state bits are changed here, make sure to update
6849	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6850	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6851	\| HM_CHANGED_GUEST_RIP
6852	\| HM_CHANGED_GUEST_RFLAGS
6853	\| HM_CHANGED_GUEST_RSP;
6854
6855	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6856	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6857	{
6858	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6859	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6860	Log4(("Clearing inhibition due to STI.\n"));
6861	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6862	}
6863	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6864
6865	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
6866	it, if we are returning to ring-3 before executing guest code. */
6867	pVCpu->hm.s.Event.fPending = false;
6868	}
6869	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6870	return rc;
6871	}
6872	else
6873	{
6874	/*
6875	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6876	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6877	*/
6878	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6879	}
6880	}
6881
6882	/* Validate. */
6883	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6884	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6885	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6886
6887	/* Inject. */
6888	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6889	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6890	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6891	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6892
6893	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6894	&& uVector == X86_XCPT_PF)
6895	{
6896	pMixedCtx->cr2 = GCPtrFaultAddress;
6897	}
6898
6899	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6900	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6901
6902	AssertRCReturn(rc, rc);
6903	return rc;
6904	}
6905
6906
6907	/**
6908	* Clears the current event in the VMCS.
6909	*
6910	* @returns VBox status code.
6911	* @param pVCpu Pointer to the VMCPU.
6912	*
6913	* @remarks Use this function only to clear events that have not yet been
6914	* delivered to the guest but are injected in the VMCS!
6915	* @remarks No-long-jump zone!!!
6916	*/
6917	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
6918	{
6919	if (!pVCpu->hm.s.Event.fPending)
6920	return;
6921
6922	#ifdef VBOX_STRICT
6923	uint32_t u32EntryInfo;
6924	int rc2 = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
6925	AssertRC(rc2);
6926	Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
6927	#endif
6928
6929	/* Clear the entry-interruption field (including the valid bit). */
6930	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
6931	AssertRC(rc);
6932
6933	/* Clear the pending debug exception field. */
6934	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
6935	AssertRC(rc);
6936	}
6937
6938
6939	/**
6940	* Enters the VT-x session.
6941	*
6942	* @returns VBox status code.
6943	* @param pVM Pointer to the VM.
6944	* @param pVCpu Pointer to the VMCPU.
6945	* @param pCpu Pointer to the CPU info struct.
6946	*/
6947	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6948	{
6949	AssertPtr(pVM);
6950	AssertPtr(pVCpu);
6951	Assert(pVM->hm.s.vmx.fSupported);
6952	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6953	NOREF(pCpu);
6954
6955	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6956	Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
6957
6958	#ifdef VBOX_STRICT
6959	/* Make sure we're in VMX root mode. */
6960	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6961	if (!(u32HostCR4 & X86_CR4_VMXE))
6962	{
6963	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6964	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6965	}
6966	#endif
6967
6968	/*
6969	* Load the VCPU's VMCS as the current (and active) one.
6970	*/
6971	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
6972	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6973	if (RT_FAILURE(rc))
6974	return rc;
6975
6976	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
6977	pVCpu->hm.s.fLeaveDone = false;
6978	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
6979
6980	return VINF_SUCCESS;
6981	}
6982
6983
6984	/**
6985	* The thread-context callback (only on platforms which support it).
6986	*
6987	* @param enmEvent The thread-context event.
6988	* @param pVCpu Pointer to the VMCPU.
6989	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
6990	* @thread EMT.
6991	*/
6992	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
6993	{
6994	switch (enmEvent)
6995	{
6996	case RTTHREADCTXEVENT_PREEMPTING:
6997	{
6998	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6999	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7000	VMCPU_ASSERT_EMT(pVCpu);
7001
7002	PVM pVM = pVCpu->CTX_SUFF(pVM);
7003	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7004
7005	/* No longjmps (logger flushes, locks) in this fragile context. */
7006	VMMRZCallRing3Disable(pVCpu);
7007	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7008
7009	/* Save the guest-state, restore host-state (FPU, debug etc.). */
7010	if (!pVCpu->hm.s.fLeaveDone)
7011	{
7012	hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
7013	pVCpu->hm.s.fLeaveDone = true;
7014	}
7015
7016	/* Leave HM context, takes care of local init (term). */
7017	int rc = HMR0LeaveCpu(pVCpu);
7018	AssertRC(rc); NOREF(rc);
7019
7020	/* Restore longjmp state. */
7021	VMMRZCallRing3Enable(pVCpu);
7022	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7023	break;
7024	}
7025
7026	case RTTHREADCTXEVENT_RESUMED:
7027	{
7028	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7029	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7030	VMCPU_ASSERT_EMT(pVCpu);
7031
7032	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7033	VMMRZCallRing3Disable(pVCpu);
7034	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7035
7036	/* Initialize the bare minimum state required for HM. This takes care of
7037	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7038	int rc = HMR0EnterCpu(pVCpu);
7039	AssertRC(rc);
7040	Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7041
7042	/* Load the active VMCS as the current one. */
7043	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7044	{
7045	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7046	AssertRC(rc); NOREF(rc);
7047	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7048	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7049	}
7050	pVCpu->hm.s.fLeaveDone = false;
7051
7052	/* Restore longjmp state. */
7053	VMMRZCallRing3Enable(pVCpu);
7054	break;
7055	}
7056
7057	default:
7058	break;
7059	}
7060	}
7061
7062
7063	/**
7064	* Saves the host state in the VMCS host-state.
7065	* Sets up the VM-exit MSR-load area.
7066	*
7067	* The CPU state will be loaded from these fields on every successful VM-exit.
7068	*
7069	* @returns VBox status code.
7070	* @param pVM Pointer to the VM.
7071	* @param pVCpu Pointer to the VMCPU.
7072	*
7073	* @remarks No-long-jump zone!!!
7074	*/
7075	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7076	{
7077	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7078
7079	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
7080	return VINF_SUCCESS;
7081
7082	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7083	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7084
7085	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7086	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7087
7088	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7089	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7090
7091	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
7092	return rc;
7093	}
7094
7095
7096	/**
7097	* Saves the host state in the VMCS host-state.
7098	*
7099	* @returns VBox status code.
7100	* @param pVM Pointer to the VM.
7101	* @param pVCpu Pointer to the VMCPU.
7102	*
7103	* @remarks No-long-jump zone!!!
7104	*/
7105	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7106	{
7107	AssertPtr(pVM);
7108	AssertPtr(pVCpu);
7109
7110	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7111
7112	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7113	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7114	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7115	return hmR0VmxSaveHostState(pVM, pVCpu);
7116	}
7117
7118
7119	/**
7120	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7121	* loaded from these fields on every successful VM-entry.
7122	*
7123	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7124	* Sets up the VM-entry controls.
7125	* Sets up the appropriate VMX non-root function to execute guest code based on
7126	* the guest CPU mode.
7127	*
7128	* @returns VBox status code.
7129	* @param pVM Pointer to the VM.
7130	* @param pVCpu Pointer to the VMCPU.
7131	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7132	* out-of-sync. Make sure to update the required fields
7133	* before using them.
7134	*
7135	* @remarks No-long-jump zone!!!
7136	*/
7137	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7138	{
7139	AssertPtr(pVM);
7140	AssertPtr(pVCpu);
7141	AssertPtr(pMixedCtx);
7142	HMVMX_ASSERT_PREEMPT_SAFE();
7143
7144	#ifdef LOG_ENABLED
7145	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7146	* probably not initialized yet? Anyway this will do for now.
7147	*
7148	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7149	* interface and disable ring-3 calls when thread-context hooks are not
7150	* available. */
7151	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7152	VMMR0LogFlushDisable(pVCpu);
7153	#endif
7154
7155	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7156
7157	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7158
7159	/* Determine real-on-v86 mode. */
7160	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7161	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7162	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7163	{
7164	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7165	}
7166
7167	/*
7168	* Load the guest-state into the VMCS.
7169	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7170	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7171	*/
7172	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7173	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7174
7175	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7176	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7177
7178	/* This needs to be done after pfnStartVM is set as it may require exit guest controls changes. */
7179	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7180	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7181
7182	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7183	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7184
7185	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7186	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7187
7188	/* Assumes CR0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7189	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7190	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7191
7192	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7193	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7194
7195	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7196	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7197
7198	/*
7199	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7200	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7201	*/
7202	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7203	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7204
7205	/* Clear any unused and reserved bits. */
7206	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7207
7208	#ifdef LOG_ENABLED
7209	/* Only reenable log-flushing if the caller has it enabled. */
7210	if (!fCallerDisabledLogFlush)
7211	VMMR0LogFlushEnable(pVCpu);
7212	#endif
7213
7214	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7215	return rc;
7216	}
7217
7218
7219	/**
7220	* Loads the state shared between the host and guest into the VMCS.
7221	*
7222	* @param pVM Pointer to the VM.
7223	* @param pVCpu Pointer to the VMCPU.
7224	* @param pCtx Pointer to the guest-CPU context.
7225	*
7226	* @remarks No-long-jump zone!!!
7227	*/
7228	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7229	{
7230	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7231	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7232
7233	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7234	{
7235	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7236	AssertRC(rc);
7237	}
7238
7239	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7240	{
7241	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7242	AssertRC(rc);
7243
7244	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7245	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
7246	{
7247	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7248	AssertRC(rc);
7249	}
7250	}
7251
7252	AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE), ("fContextUseFlags=%#x\n",
7253	pVCpu->hm.s.fContextUseFlags));
7254	}
7255
7256
7257	/**
7258	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7259	*
7260	* @param pVM Pointer to the VM.
7261	* @param pVCpu Pointer to the VMCPU.
7262	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7263	* out-of-sync. Make sure to update the required fields
7264	* before using them.
7265	*/
7266	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7267	{
7268	HMVMX_ASSERT_PREEMPT_SAFE();
7269
7270	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7271	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7272	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
7273	#endif
7274
7275	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7276	{
7277	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7278	AssertRC(rc);
7279	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7280	}
7281	else if (pVCpu->hm.s.fContextUseFlags)
7282	{
7283	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7284	AssertRC(rc);
7285	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7286	}
7287
7288	/* All the guest state bits should be loaded except maybe the host context and shared host/guest bits. */
7289	AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7290	\|\| !(pVCpu->hm.s.fContextUseFlags & ~(HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE)),
7291	("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7292
7293	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7294	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7295	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7296	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7297	#endif
7298	}
7299
7300
7301	/**
7302	* Does the preparations before executing guest code in VT-x.
7303	*
7304	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7305	* recompiler. We must be cautious what we do here regarding committing
7306	* guest-state information into the VMCS assuming we assuredly execute the
7307	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7308	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7309	* so that the recompiler can (and should) use them when it resumes guest
7310	* execution. Otherwise such operations must be done when we can no longer
7311	* exit to ring-3.
7312	*
7313	* @returns Strict VBox status code.
7314	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7315	* have been disabled.
7316	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7317	* double-fault into the guest.
7318	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7319	*
7320	* @param pVM Pointer to the VM.
7321	* @param pVCpu Pointer to the VMCPU.
7322	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7323	* out-of-sync. Make sure to update the required fields
7324	* before using them.
7325	* @param pVmxTransient Pointer to the VMX transient structure.
7326	*
7327	* @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7328	* interrupts will be disabled.
7329	*/
7330	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7331	{
7332	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7333
7334	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7335	PGMRZDynMapFlushAutoSet(pVCpu);
7336	#endif
7337
7338	/* Check force flag actions that might require us to go back to ring-3. */
7339	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7340	if (rc != VINF_SUCCESS)
7341	return rc;
7342
7343	#ifndef IEM_VERIFICATION_MODE_FULL
7344	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7345	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7346	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7347	{
7348	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7349	RTGCPHYS GCPhysApicBase;
7350	GCPhysApicBase = pMixedCtx->msrApicBase;
7351	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7352
7353	/* Unalias any existing mapping. */
7354	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7355	AssertRCReturn(rc, rc);
7356
7357	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7358	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7359	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7360	AssertRCReturn(rc, rc);
7361
7362	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7363	}
7364	#endif /* !IEM_VERIFICATION_MODE_FULL */
7365
7366	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7367	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7368
7369	/*
7370	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7371	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7372	*/
7373	if (TRPMHasTrap(pVCpu))
7374	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7375	else if (!pVCpu->hm.s.Event.fPending)
7376	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7377
7378	/*
7379	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7380	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7381	*/
7382	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7383	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7384	{
7385	Assert(rc == VINF_EM_RESET);
7386	return rc;
7387	}
7388
7389	/*
7390	* No longjmps to ring-3 from this point on!!!
7391	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7392	* This also disables flushing of the R0-logger instance (if any).
7393	*/
7394	VMMRZCallRing3Disable(pVCpu);
7395
7396	/*
7397	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7398	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7399	*
7400	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7401	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7402	*
7403	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7404	* executing guest code.
7405	*/
7406	pVmxTransient->uEflags = ASMIntDisableFlags();
7407	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7408	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7409	{
7410	hmR0VmxClearEventVmcs(pVCpu);
7411	ASMSetFlags(pVmxTransient->uEflags);
7412	VMMRZCallRing3Enable(pVCpu);
7413	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7414	return VINF_EM_RAW_TO_R3;
7415	}
7416	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7417	{
7418	hmR0VmxClearEventVmcs(pVCpu);
7419	ASMSetFlags(pVmxTransient->uEflags);
7420	VMMRZCallRing3Enable(pVCpu);
7421	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7422	return VINF_EM_RAW_INTERRUPT;
7423	}
7424
7425	/* We've injected any pending events. This is really the point of no return (to ring-3). */
7426	pVCpu->hm.s.Event.fPending = false;
7427
7428	return VINF_SUCCESS;
7429	}
7430
7431
7432	/**
7433	* Prepares to run guest code in VT-x and we've committed to doing so. This
7434	* means there is no backing out to ring-3 or anywhere else at this
7435	* point.
7436	*
7437	* @param pVM Pointer to the VM.
7438	* @param pVCpu Pointer to the VMCPU.
7439	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7440	* out-of-sync. Make sure to update the required fields
7441	* before using them.
7442	* @param pVmxTransient Pointer to the VMX transient structure.
7443	*
7444	* @remarks Called with preemption disabled.
7445	* @remarks No-long-jump zone!!!
7446	*/
7447	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7448	{
7449	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7450	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7451	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7452
7453	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7454	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7455
7456	/*
7457	* If we are injecting events to a real-on-v86 mode guest, we may have to update
7458	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7459	* Reload only the necessary state, the assertion will catch if other parts of the code
7460	* change.
7461	*/
7462	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7463	{
7464	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7465	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7466	}
7467
7468	/*
7469	* Load the host state bits as we may've been preempted (only happens when
7470	* thread-context hooks are used).
7471	*/
7472	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7473	{
7474	/* This ASSUMES that pfnStartVM has been set up already. */
7475	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7476	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7477	AssertRC(rc);
7478	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7479	}
7480	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7481
7482	/*
7483	* Load the state shared between host and guest (FPU, debug).
7484	*/
7485	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE)
7486	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7487	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7488
7489	/*
7490	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7491	*/
7492	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7493	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7494
7495	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7496	RTCPUID idCurrentCpu = pCpu->idCpu;
7497	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7498	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
7499	{
7500	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7501	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7502	}
7503
7504	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7505	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7506	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7507	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7508
7509	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7510
7511	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7512	to start executing. */
7513
7514	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7515	/*
7516	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7517	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7518	*/
7519	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7520	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7521	{
7522	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7523	uint64_t u64HostTscAux = 0;
7524	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7525	AssertRC(rc2);
7526	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7527	}
7528	#endif
7529	}
7530
7531
7532	/**
7533	* Performs some essential restoration of state after running guest code in
7534	* VT-x.
7535	*
7536	* @param pVM Pointer to the VM.
7537	* @param pVCpu Pointer to the VMCPU.
7538	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7539	* out-of-sync. Make sure to update the required fields
7540	* before using them.
7541	* @param pVmxTransient Pointer to the VMX transient structure.
7542	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7543	*
7544	* @remarks Called with interrupts disabled, and returns with interrups enabled!
7545	*
7546	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7547	* unconditionally when it is safe to do so.
7548	*/
7549	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7550	{
7551	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7552
7553	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7554	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7555	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7556	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7557	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7558
7559	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7560	{
7561	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7562	/* Restore host's TSC_AUX. */
7563	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7564	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7565	#endif
7566	/** @todo Find a way to fix hardcoding a guestimate. */
7567	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7568	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7569	}
7570
7571	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7572	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7573	Assert(!(ASMGetFlags() & X86_EFL_IF));
7574	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7575
7576	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7577	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7578	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7579	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7580
7581	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7582	uint32_t uExitReason;
7583	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7584	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7585	AssertRC(rc);
7586	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7587	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7588
7589	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7590	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7591	{
7592	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7593	pVmxTransient->fVMEntryFailed));
7594	return;
7595	}
7596
7597	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7598	{
7599	/* Update the guest interruptibility-state from the VMCS. */
7600	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7601	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7602	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7603	AssertRC(rc);
7604	#endif
7605	/*
7606	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7607	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7608	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7609	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7610	*/
7611	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7612	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7613	{
7614	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7615	AssertRC(rc);
7616	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7617	}
7618	}
7619	}
7620
7621
7622
7623	/**
7624	* Runs the guest code using VT-x the normal way.
7625	*
7626	* @returns VBox status code.
7627	* @param pVM Pointer to the VM.
7628	* @param pVCpu Pointer to the VMCPU.
7629	* @param pCtx Pointer to the guest-CPU context.
7630	*
7631	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
7632	* @remarks Called with preemption disabled.
7633	*/
7634	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7635	{
7636	VMXTRANSIENT VmxTransient;
7637	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7638	int rc = VERR_INTERNAL_ERROR_5;
7639	uint32_t cLoops = 0;
7640
7641	for (;; cLoops++)
7642	{
7643	Assert(!HMR0SuspendPending());
7644	HMVMX_ASSERT_CPU_SAFE();
7645
7646	/* Preparatory work for running guest code, this may force us to return
7647	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7648	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7649	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7650	if (rc != VINF_SUCCESS)
7651	break;
7652
7653	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7654	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7655	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7656
7657	/* Restore any residual host-state and save any bits shared between host
7658	and guest into the guest-CPU state. Re-enables interrupts! */
7659	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7660
7661	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7662	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7663	{
7664	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7665	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7666	return rc;
7667	}
7668
7669	/* Handle the VM-exit. */
7670	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7671	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7672	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7673	HMVMX_START_EXIT_DISPATCH_PROF();
7674	#ifdef HMVMX_USE_FUNCTION_TABLE
7675	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7676	#else
7677	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7678	#endif
7679	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7680	if (rc != VINF_SUCCESS)
7681	break;
7682	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7683	{
7684	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7685	rc = VINF_EM_RAW_INTERRUPT;
7686	break;
7687	}
7688	}
7689
7690	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7691	return rc;
7692	}
7693
7694
7695	/**
7696	* Single steps guest code using VT-x.
7697	*
7698	* @returns VBox status code.
7699	* @param pVM Pointer to the VM.
7700	* @param pVCpu Pointer to the VMCPU.
7701	* @param pCtx Pointer to the guest-CPU context.
7702	*
7703	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7704	* @remarks Called with preemption disabled.
7705	*/
7706	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7707	{
7708	VMXTRANSIENT VmxTransient;
7709	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7710	int rc = VERR_INTERNAL_ERROR_5;
7711	uint32_t cLoops = 0;
7712	uint16_t uCsStart = pCtx->cs.Sel;
7713	uint64_t uRipStart = pCtx->rip;
7714
7715	for (;; cLoops++)
7716	{
7717	Assert(!HMR0SuspendPending());
7718	HMVMX_ASSERT_CPU_SAFE();
7719
7720	/* Preparatory work for running guest code, this may force us to return
7721	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7722	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7723	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7724	if (rc != VINF_SUCCESS)
7725	break;
7726
7727	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7728	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7729	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7730
7731	/* Restore any residual host-state and save any bits shared between host
7732	and guest into the guest-CPU state. Re-enables interrupts! */
7733	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7734
7735	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7736	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7737	{
7738	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7739	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7740	return rc;
7741	}
7742
7743	/* Handle the VM-exit. */
7744	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7745	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7746	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7747	HMVMX_START_EXIT_DISPATCH_PROF();
7748	#ifdef HMVMX_USE_FUNCTION_TABLE
7749	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7750	#else
7751	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7752	#endif
7753	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7754	if (rc != VINF_SUCCESS)
7755	break;
7756	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7757	{
7758	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7759	rc = VINF_EM_RAW_INTERRUPT;
7760	break;
7761	}
7762
7763	/*
7764	* Did the RIP change, if so, consider it a single step.
7765	* Otherwise, make sure one of the TFs gets set.
7766	*/
7767	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7768	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7769	AssertRCReturn(rc2, rc2);
7770	if ( pCtx->rip != uRipStart
7771	\|\| pCtx->cs.Sel != uCsStart)
7772	{
7773	rc = VINF_EM_DBG_STEPPED;
7774	break;
7775	}
7776	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7777	}
7778
7779	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7780	return rc;
7781	}
7782
7783
7784	/**
7785	* Runs the guest code using VT-x.
7786	*
7787	* @returns VBox status code.
7788	* @param pVM Pointer to the VM.
7789	* @param pVCpu Pointer to the VMCPU.
7790	* @param pCtx Pointer to the guest-CPU context.
7791	*
7792	* @remarks Called with preemption disabled.
7793	*/
7794	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7795	{
7796	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7797	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7798	HMVMX_ASSERT_PREEMPT_SAFE();
7799
7800	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7801
7802	int rc;
7803	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7804	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7805	else
7806	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7807
7808	if (rc == VERR_EM_INTERPRETER)
7809	rc = VINF_EM_RAW_EMULATE_INSTR;
7810	else if (rc == VINF_EM_RESET)
7811	rc = VINF_EM_TRIPLE_FAULT;
7812
7813	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7814	if (RT_FAILURE(rc2))
7815	{
7816	pVCpu->hm.s.u32HMError = rc;
7817	rc = rc2;
7818	}
7819	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7820	return rc;
7821	}
7822
7823
7824	#ifndef HMVMX_USE_FUNCTION_TABLE
7825	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7826	{
7827	int rc;
7828	switch (rcReason)
7829	{
7830	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7831	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7832	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7833	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7834	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7835	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7836	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7837	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7838	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7839	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7840	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7841	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7842	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7843	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7844	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7845	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7846	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7847	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7848	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7849	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7850	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7851	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7852	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7853	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7854	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7855	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7856	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7857	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7858	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7859	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7860	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7861	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7862	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7863
7864	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7865	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7866	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7867	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7868	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7869	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7870	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7871	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7872	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7873
7874	case VMX_EXIT_VMCALL:
7875	case VMX_EXIT_VMCLEAR:
7876	case VMX_EXIT_VMLAUNCH:
7877	case VMX_EXIT_VMPTRLD:
7878	case VMX_EXIT_VMPTRST:
7879	case VMX_EXIT_VMREAD:
7880	case VMX_EXIT_VMRESUME:
7881	case VMX_EXIT_VMWRITE:
7882	case VMX_EXIT_VMXOFF:
7883	case VMX_EXIT_VMXON:
7884	case VMX_EXIT_INVEPT:
7885	case VMX_EXIT_INVVPID:
7886	case VMX_EXIT_VMFUNC:
7887	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7888	break;
7889	default:
7890	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7891	break;
7892	}
7893	return rc;
7894	}
7895	#endif
7896
7897	#ifdef DEBUG
7898	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7899	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7900	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7901
7902	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7903	do \
7904	{ \
7905	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7906	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7907	} while (0)
7908
7909	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7910	do { \
7911	AssertPtr(pVCpu); \
7912	AssertPtr(pMixedCtx); \
7913	AssertPtr(pVmxTransient); \
7914	Assert(pVmxTransient->fVMEntryFailed == false); \
7915	Assert(ASMIntAreEnabled()); \
7916	HMVMX_ASSERT_PREEMPT_SAFE(); \
7917	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7918	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7919	HMVMX_ASSERT_PREEMPT_SAFE(); \
7920	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7921	HMVMX_ASSERT_PREEMPT_CPUID(); \
7922	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7923	} while (0)
7924
7925	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7926	do { \
7927	Log4Func(("\n")); \
7928	} while(0)
7929	#else /* Release builds */
7930	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7931	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7932	#endif
7933
7934
7935	/**
7936	* Advances the guest RIP after reading it from the VMCS.
7937	*
7938	* @returns VBox status code.
7939	* @param pVCpu Pointer to the VMCPU.
7940	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7941	* out-of-sync. Make sure to update the required fields
7942	* before using them.
7943	* @param pVmxTransient Pointer to the VMX transient structure.
7944	*
7945	* @remarks No-long-jump zone!!!
7946	*/
7947	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7948	{
7949	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7950	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7951	AssertRCReturn(rc, rc);
7952
7953	pMixedCtx->rip += pVmxTransient->cbInstr;
7954	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7955	return rc;
7956	}
7957
7958
7959	/**
7960	* Tries to determine what part of the guest-state VT-x has deemed as invalid
7961	* and update error record fields accordingly.
7962	*
7963	* @return VMX_IGS_* return codes.
7964	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7965	* wrong with the guest state.
7966	*
7967	* @param pVM Pointer to the VM.
7968	* @param pVCpu Pointer to the VMCPU.
7969	* @param pCtx Pointer to the guest-CPU state.
7970	*/
7971	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7972	{
7973	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7974	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7975	uError = (err); \
7976	break; \
7977	} else do {} while (0)
7978	/* Duplicate of IEM_IS_CANONICAL(). */
7979	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7980
7981	int rc;
7982	uint64_t u64Val;
7983	uint32_t u32Val;
7984	uint32_t uError = VMX_IGS_ERROR;
7985	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7986
7987	do
7988	{
7989	/*
7990	* CR0.
7991	*/
7992	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7993	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7994	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7995	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7996	if (fUnrestrictedGuest)
7997	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
7998
7999	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8000	AssertRCBreak(rc);
8001	HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8002	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8003	if ( !fUnrestrictedGuest
8004	&& (u32Val & X86_CR0_PG)
8005	&& !(u32Val & X86_CR0_PE))
8006	{
8007	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8008	}
8009
8010	/*
8011	* CR4.
8012	*/
8013	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8014	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8015	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8016	AssertRCBreak(rc);
8017	HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8018	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8019
8020	/*
8021	* IA32_DEBUGCTL MSR.
8022	*/
8023	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8024	AssertRCBreak(rc);
8025	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8026	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8027	{
8028	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8029	}
8030	uint64_t u64DebugCtlMsr = u64Val;
8031
8032	#ifdef VBOX_STRICT
8033	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
8034	AssertRCBreak(rc);
8035	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8036	#endif
8037	bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8038
8039	/*
8040	* RIP and RFLAGS.
8041	*/
8042	uint32_t u32Eflags;
8043	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8044	if (HMVMX_IS_64BIT_HOST_MODE())
8045	{
8046	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8047	AssertRCBreak(rc);
8048	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8049	if ( !fLongModeGuest
8050	\|\| !pCtx->cs.Attr.n.u1Long)
8051	{
8052	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8053	}
8054	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8055	* must be identical if the "IA32e mode guest" VM-entry control is 1
8056	* and CS.L is 1. No check applies if the CPU supports 64
8057	* linear-address bits. */
8058
8059	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8060	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8061	AssertRCBreak(rc);
8062	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8063	VMX_IGS_RFLAGS_RESERVED);
8064	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8065	u32Eflags = u64Val;
8066	}
8067	else
8068	#endif
8069	{
8070	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8071	AssertRCBreak(rc);
8072	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8073	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8074	}
8075
8076	if ( fLongModeGuest
8077	\|\| !(pCtx->cr0 & X86_CR0_PE))
8078	{
8079	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8080	}
8081
8082	uint32_t u32EntryInfo;
8083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8084	AssertRCBreak(rc);
8085	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8086	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8087	{
8088	HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8089	}
8090
8091	/*
8092	* 64-bit checks.
8093	*/
8094	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8095	if (HMVMX_IS_64BIT_HOST_MODE())
8096	{
8097	if ( fLongModeGuest
8098	&& !fUnrestrictedGuest)
8099	{
8100	HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
8101	HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
8102	}
8103
8104	if ( !fLongModeGuest
8105	&& (pCtx->cr4 & X86_CR4_PCIDE))
8106	{
8107	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8108	}
8109
8110	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8111	* 51:32 beyond the processor's physical-address width are 0. */
8112
8113	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8114	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8115	{
8116	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8117	}
8118
8119	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8120	AssertRCBreak(rc);
8121	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8122
8123	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8124	AssertRCBreak(rc);
8125	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8126	}
8127	#endif
8128
8129	/*
8130	* PERF_GLOBAL MSR.
8131	*/
8132	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8133	{
8134	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8135	AssertRCBreak(rc);
8136	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8137	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8138	}
8139
8140	/*
8141	* PAT MSR.
8142	*/
8143	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8144	{
8145	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8146	AssertRCBreak(rc);
8147	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8148	for (unsigned i = 0; i < 8; i++)
8149	{
8150	uint8_t u8Val = (u64Val & 0x7);
8151	if ( u8Val != 0 /* UC */
8152	\|\| u8Val != 1 /* WC */
8153	\|\| u8Val != 4 /* WT */
8154	\|\| u8Val != 5 /* WP */
8155	\|\| u8Val != 6 /* WB */
8156	\|\| u8Val != 7 /* UC- */)
8157	{
8158	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8159	}
8160	u64Val >>= 3;
8161	}
8162	}
8163
8164	/*
8165	* EFER MSR.
8166	*/
8167	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8168	{
8169	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8170	AssertRCBreak(rc);
8171	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8172	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8173	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8174	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8175	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8176	\|\| (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8177	}
8178
8179	/*
8180	* Segment registers.
8181	*/
8182	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8183	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8184	if (!(u32Eflags & X86_EFL_VM))
8185	{
8186	/* CS */
8187	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8188	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8189	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8190	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8191	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8192	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8193	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8194	/* CS cannot be loaded with NULL in protected mode. */
8195	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8196	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8197	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8198	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8199	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8200	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8201	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8202	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8203	else
8204	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8205
8206	/* SS */
8207	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8208	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8209	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8210	if ( !(pCtx->cr0 & X86_CR0_PE)
8211	\|\| pCtx->cs.Attr.n.u4Type == 3)
8212	{
8213	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8214	}
8215	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8216	{
8217	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8218	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8219	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8220	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8221	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8222	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8223	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8224	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8225	}
8226
8227	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8228	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8229	{
8230	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8231	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8232	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8233	\|\| pCtx->ds.Attr.n.u4Type > 11
8234	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8235	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8236	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8237	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8238	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8239	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8240	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8241	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8242	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8243	}
8244	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8245	{
8246	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8247	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8248	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8249	\|\| pCtx->es.Attr.n.u4Type > 11
8250	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8251	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8252	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8253	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8254	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8255	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8256	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8257	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8258	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8259	}
8260	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8261	{
8262	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8263	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8264	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8265	\|\| pCtx->fs.Attr.n.u4Type > 11
8266	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8267	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8268	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8269	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8270	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8271	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8272	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8273	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8274	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8275	}
8276	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8277	{
8278	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8279	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8280	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8281	\|\| pCtx->gs.Attr.n.u4Type > 11
8282	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8283	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8284	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8285	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8286	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8287	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8288	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8289	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8290	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8291	}
8292	/* 64-bit capable CPUs. */
8293	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8294	if (HMVMX_IS_64BIT_HOST_MODE())
8295	{
8296	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8297	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8298	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8299	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8300	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8301	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8302	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8303	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8304	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8305	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8306	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8307	}
8308	#endif
8309	}
8310	else
8311	{
8312	/* V86 mode checks. */
8313	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8314	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8315	{
8316	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8317	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8318	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8319	}
8320	else
8321	{
8322	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8323	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8324	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8325	}
8326
8327	/* CS */
8328	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8329	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8330	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8331	/* SS */
8332	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8333	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8334	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8335	/* DS */
8336	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8337	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8338	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8339	/* ES */
8340	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8341	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8342	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8343	/* FS */
8344	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8345	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8346	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8347	/* GS */
8348	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8349	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8350	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8351	/* 64-bit capable CPUs. */
8352	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8353	if (HMVMX_IS_64BIT_HOST_MODE())
8354	{
8355	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8356	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8357	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8358	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8359	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8360	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8361	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8362	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8363	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8364	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8365	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8366	}
8367	#endif
8368	}
8369
8370	/*
8371	* TR.
8372	*/
8373	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8374	/* 64-bit capable CPUs. */
8375	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8376	if (HMVMX_IS_64BIT_HOST_MODE())
8377	{
8378	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8379	}
8380	#endif
8381	if (fLongModeGuest)
8382	{
8383	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8384	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8385	}
8386	else
8387	{
8388	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8389	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8390	VMX_IGS_TR_ATTR_TYPE_INVALID);
8391	}
8392	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8393	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8394	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8395	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8396	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8397	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8398	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8399	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8400
8401	/*
8402	* GDTR and IDTR.
8403	*/
8404	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8405	if (HMVMX_IS_64BIT_HOST_MODE())
8406	{
8407	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8408	AssertRCBreak(rc);
8409	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8410
8411	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8412	AssertRCBreak(rc);
8413	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8414	}
8415	#endif
8416
8417	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8418	AssertRCBreak(rc);
8419	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8420
8421	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8422	AssertRCBreak(rc);
8423	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8424
8425	/*
8426	* Guest Non-Register State.
8427	*/
8428	/* Activity State. */
8429	uint32_t u32ActivityState;
8430	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8431	AssertRCBreak(rc);
8432	HMVMX_CHECK_BREAK( !u32ActivityState
8433	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8434	VMX_IGS_ACTIVITY_STATE_INVALID);
8435	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8436	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8437	uint32_t u32IntrState;
8438	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8439	AssertRCBreak(rc);
8440	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8441	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8442	{
8443	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8444	}
8445
8446	/** @todo Activity state and injecting interrupts. Left as a todo since we
8447	* currently don't use activity states but ACTIVE. */
8448
8449	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8450	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8451
8452	/* Guest interruptibility-state. */
8453	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8454	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8455	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8456	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8457	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8458	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8459	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8460	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8461	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8462	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8463	{
8464	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8465	{
8466	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8467	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8468	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8469	}
8470	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8471	{
8472	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8473	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8474	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8475	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8476	}
8477	}
8478	/** @todo Assumes the processor is not in SMM. */
8479	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8480	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8481	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8482	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8483	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8484	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8485	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8486	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8487	{
8488	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8489	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8490	}
8491
8492	/* Pending debug exceptions. */
8493	if (HMVMX_IS_64BIT_HOST_MODE())
8494	{
8495	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8496	AssertRCBreak(rc);
8497	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8498	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8499	u32Val = u64Val; /* For pending debug exceptions checks below. */
8500	}
8501	else
8502	{
8503	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8504	AssertRCBreak(rc);
8505	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8506	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8507	}
8508
8509	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8510	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8511	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8512	{
8513	if ( (u32Eflags & X86_EFL_TF)
8514	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8515	{
8516	/* Bit 14 is PendingDebug.BS. */
8517	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8518	}
8519	if ( !(u32Eflags & X86_EFL_TF)
8520	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8521	{
8522	/* Bit 14 is PendingDebug.BS. */
8523	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8524	}
8525	}
8526
8527	/* VMCS link pointer. */
8528	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8529	AssertRCBreak(rc);
8530	if (u64Val != UINT64_C(0xffffffffffffffff))
8531	{
8532	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8533	/** @todo Bits beyond the processor's physical-address width MBZ. */
8534	/** @todo 32-bit located in memory referenced by value of this field (as a
8535	* physical address) must contain the processor's VMCS revision ID. */
8536	/** @todo SMM checks. */
8537	}
8538
8539	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8540
8541	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8542	if (uError == VMX_IGS_ERROR)
8543	uError = VMX_IGS_REASON_NOT_FOUND;
8544	} while (0);
8545
8546	pVCpu->hm.s.u32HMError = uError;
8547	return uError;
8548
8549	#undef HMVMX_ERROR_BREAK
8550	#undef HMVMX_CHECK_BREAK
8551	#undef HMVMX_IS_CANONICAL
8552	}
8553
8554	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8555	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8556	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8557
8558	/** @name VM-exit handlers.
8559	* @{
8560	*/
8561
8562	/**
8563	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8564	*/
8565	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8566	{
8567	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8569	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8570	#if HC_ARCH_BITS == 64
8571	Assert(ASMIntAreEnabled());
8572	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8573	return VINF_SUCCESS;
8574	#endif
8575	return VINF_EM_RAW_INTERRUPT;
8576	}
8577
8578
8579	/**
8580	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8581	*/
8582	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8583	{
8584	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8585	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8586
8587	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8588	AssertRCReturn(rc, rc);
8589
8590	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8591	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8592	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8593	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8594
8595	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8596	{
8597	/*
8598	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8599	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8600	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8601	*
8602	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8603	*/
8604	VMXDispatchHostNmi();
8605	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8606	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8607	return VINF_SUCCESS;
8608	}
8609
8610	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8611	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8612	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8613	{
8614	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8615	return VINF_SUCCESS;
8616	}
8617	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8618	{
8619	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8620	return rc;
8621	}
8622
8623	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8624	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8625	switch (uIntrType)
8626	{
8627	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8628	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8629	/* no break */
8630	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8631	{
8632	switch (uVector)
8633	{
8634	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8635	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8636	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8637	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8638	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8639	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8640	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8641	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8642	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8643	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8644	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8645	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8646	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8647	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8648	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8649	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8650	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8651	#endif
8652	default:
8653	{
8654	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8655	AssertRCReturn(rc, rc);
8656
8657	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8658	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8659	{
8660	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8661	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8662	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8663
8664	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8665	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8666	AssertRCReturn(rc, rc);
8667	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8668	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8669	0 /* GCPtrFaultAddress */);
8670	AssertRCReturn(rc, rc);
8671	}
8672	else
8673	{
8674	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8675	pVCpu->hm.s.u32HMError = uVector;
8676	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8677	}
8678	break;
8679	}
8680	}
8681	break;
8682	}
8683
8684	default:
8685	{
8686	pVCpu->hm.s.u32HMError = uExitIntrInfo;
8687	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8688	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8689	break;
8690	}
8691	}
8692	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8693	return rc;
8694	}
8695
8696
8697	/**
8698	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8699	*/
8700	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8701	{
8702	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8703
8704	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8705	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8706	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8707	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8708	AssertRCReturn(rc, rc);
8709
8710	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8711	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8712	return VINF_SUCCESS;
8713	}
8714
8715
8716	/**
8717	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8718	*/
8719	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8720	{
8721	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8722	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8723	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8724	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8725	}
8726
8727
8728	/**
8729	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8730	*/
8731	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8732	{
8733	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8734	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8735	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8736	}
8737
8738
8739	/**
8740	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8741	*/
8742	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8743	{
8744	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8745	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8746	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8747	}
8748
8749
8750	/**
8751	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8752	*/
8753	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8754	{
8755	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8756	PVM pVM = pVCpu->CTX_SUFF(pVM);
8757	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8758	if (RT_LIKELY(rc == VINF_SUCCESS))
8759	{
8760	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8761	Assert(pVmxTransient->cbInstr == 2);
8762	}
8763	else
8764	{
8765	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8766	rc = VERR_EM_INTERPRETER;
8767	}
8768	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8769	return rc;
8770	}
8771
8772
8773	/**
8774	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8775	*/
8776	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8777	{
8778	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8779	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8780	AssertRCReturn(rc, rc);
8781
8782	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8783	return VINF_EM_RAW_EMULATE_INSTR;
8784
8785	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8786	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8787	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8788	}
8789
8790
8791	/**
8792	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8793	*/
8794	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8795	{
8796	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8797	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8798	AssertRCReturn(rc, rc);
8799
8800	PVM pVM = pVCpu->CTX_SUFF(pVM);
8801	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8802	if (RT_LIKELY(rc == VINF_SUCCESS))
8803	{
8804	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8805	Assert(pVmxTransient->cbInstr == 2);
8806	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8807	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8808	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8809	}
8810	else
8811	{
8812	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8813	rc = VERR_EM_INTERPRETER;
8814	}
8815	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8816	return rc;
8817	}
8818
8819
8820	/**
8821	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8822	*/
8823	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8824	{
8825	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8826	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8827	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8828	AssertRCReturn(rc, rc);
8829
8830	PVM pVM = pVCpu->CTX_SUFF(pVM);
8831	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8832	if (RT_LIKELY(rc == VINF_SUCCESS))
8833	{
8834	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8835	Assert(pVmxTransient->cbInstr == 3);
8836	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8837	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8838	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8839	}
8840	else
8841	{
8842	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8843	rc = VERR_EM_INTERPRETER;
8844	}
8845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8846	return rc;
8847	}
8848
8849
8850	/**
8851	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8852	*/
8853	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8854	{
8855	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8856	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8857	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8858	AssertRCReturn(rc, rc);
8859
8860	PVM pVM = pVCpu->CTX_SUFF(pVM);
8861	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8862	if (RT_LIKELY(rc == VINF_SUCCESS))
8863	{
8864	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8865	Assert(pVmxTransient->cbInstr == 2);
8866	}
8867	else
8868	{
8869	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8870	rc = VERR_EM_INTERPRETER;
8871	}
8872	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8873	return rc;
8874	}
8875
8876
8877	/**
8878	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8879	*/
8880	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8881	{
8882	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8883	PVM pVM = pVCpu->CTX_SUFF(pVM);
8884	Assert(!pVM->hm.s.fNestedPaging);
8885
8886	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8887	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8888	AssertRCReturn(rc, rc);
8889
8890	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8891	rc = VBOXSTRICTRC_VAL(rc2);
8892	if (RT_LIKELY(rc == VINF_SUCCESS))
8893	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8894	else
8895	{
8896	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8897	pVmxTransient->uExitQualification, rc));
8898	}
8899	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8900	return rc;
8901	}
8902
8903
8904	/**
8905	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8906	*/
8907	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8908	{
8909	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8910	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8911	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8912	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8913	AssertRCReturn(rc, rc);
8914
8915	PVM pVM = pVCpu->CTX_SUFF(pVM);
8916	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8917	if (RT_LIKELY(rc == VINF_SUCCESS))
8918	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8919	else
8920	{
8921	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8922	rc = VERR_EM_INTERPRETER;
8923	}
8924	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8925	return rc;
8926	}
8927
8928
8929	/**
8930	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8931	*/
8932	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8933	{
8934	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8935	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8936	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8937	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8938	AssertRCReturn(rc, rc);
8939
8940	PVM pVM = pVCpu->CTX_SUFF(pVM);
8941	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8942	rc = VBOXSTRICTRC_VAL(rc2);
8943	if (RT_LIKELY( rc == VINF_SUCCESS
8944	\|\| rc == VINF_EM_HALT))
8945	{
8946	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8947	AssertRCReturn(rc3, rc3);
8948
8949	if ( rc == VINF_EM_HALT
8950	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8951	{
8952	rc = VINF_SUCCESS;
8953	}
8954	}
8955	else
8956	{
8957	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8958	rc = VERR_EM_INTERPRETER;
8959	}
8960	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
8961	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8962	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8963	return rc;
8964	}
8965
8966
8967	/**
8968	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8969	*/
8970	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8971	{
8972	/*
8973	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8974	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8975	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
8976	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8977	*/
8978	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8979	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8980	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8981	}
8982
8983
8984	/**
8985	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8986	*/
8987	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8988	{
8989	/*
8990	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8991	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8992	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
8993	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8994	*/
8995	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8996	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8997	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8998	}
8999
9000
9001	/**
9002	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9003	*/
9004	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9005	{
9006	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9007	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9008	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
9009	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9010	}
9011
9012
9013	/**
9014	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9015	*/
9016	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9017	{
9018	/*
9019	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9020	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9021	* See Intel spec. 25.3 "Other Causes of VM-exits".
9022	*/
9023	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9024	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
9025	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9026	}
9027
9028
9029	/**
9030	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9031	* VM-exit.
9032	*/
9033	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9034	{
9035	/*
9036	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9037	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9038	*
9039	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9040	* See Intel spec. "23.8 Restrictions on VMX operation".
9041	*/
9042	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9043	return VINF_SUCCESS;
9044	}
9045
9046
9047	/**
9048	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9049	* VM-exit.
9050	*/
9051	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9052	{
9053	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9054	return VINF_EM_RESET;
9055	}
9056
9057
9058	/**
9059	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9060	*/
9061	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9062	{
9063	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9064	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9065	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9066	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9067	AssertRCReturn(rc, rc);
9068
9069	pMixedCtx->rip++;
9070	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9071	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9072	rc = VINF_SUCCESS;
9073	else
9074	rc = VINF_EM_HALT;
9075
9076	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9077	return rc;
9078	}
9079
9080
9081	/**
9082	* VM-exit handler for instructions that result in a #UD exception delivered to
9083	* the guest.
9084	*/
9085	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9086	{
9087	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9088	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9089	return VINF_SUCCESS;
9090	}
9091
9092
9093	/**
9094	* VM-exit handler for expiry of the VMX preemption timer.
9095	*/
9096	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9097	{
9098	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9099
9100	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9101	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9102
9103	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9104	PVM pVM = pVCpu->CTX_SUFF(pVM);
9105	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9107	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9108	}
9109
9110
9111	/**
9112	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9113	*/
9114	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9115	{
9116	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9117
9118	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9119	/** @todo check if XSETBV is supported by the recompiler. */
9120	return VERR_EM_INTERPRETER;
9121	}
9122
9123
9124	/**
9125	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9126	*/
9127	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9128	{
9129	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9130
9131	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9132	/** @todo implement EMInterpretInvpcid() */
9133	return VERR_EM_INTERPRETER;
9134	}
9135
9136
9137	/**
9138	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9139	* Error VM-exit.
9140	*/
9141	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9142	{
9143	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9144	AssertRCReturn(rc, rc);
9145
9146	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9147	NOREF(uInvalidReason);
9148
9149	#ifdef VBOX_STRICT
9150	uint32_t uIntrState;
9151	HMVMXHCUINTREG uHCReg;
9152	uint64_t u64Val;
9153	uint32_t u32Val;
9154
9155	rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9156	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9157	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9158	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9159	AssertRCReturn(rc, rc);
9160
9161	Log4(("uInvalidReason %u\n", uInvalidReason));
9162	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9163	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9164	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9165	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9166
9167	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9168	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9169	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9170	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9171	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9172	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9173	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9174	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9175	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9176	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9177	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9178	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9179	#endif
9180
9181	PVM pVM = pVCpu->CTX_SUFF(pVM);
9182	HMDumpRegs(pVM, pVCpu, pMixedCtx);
9183
9184	return VERR_VMX_INVALID_GUEST_STATE;
9185	}
9186
9187
9188	/**
9189	* VM-exit handler for VM-entry failure due to an MSR-load
9190	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9191	*/
9192	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9193	{
9194	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9195	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9196	}
9197
9198
9199	/**
9200	* VM-exit handler for VM-entry failure due to a machine-check event
9201	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9202	*/
9203	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9204	{
9205	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9206	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9207	}
9208
9209
9210	/**
9211	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9212	* theory.
9213	*/
9214	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9215	{
9216	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9217	return VERR_VMX_UNDEFINED_EXIT_CODE;
9218	}
9219
9220
9221	/**
9222	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9223	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9224	* Conditional VM-exit.
9225	*/
9226	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9227	{
9228	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9229
9230	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9232	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9233	return VERR_EM_INTERPRETER;
9234	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9235	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9236	}
9237
9238
9239	/**
9240	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9241	*/
9242	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9243	{
9244	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9245
9246	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9247	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9248	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9249	return VERR_EM_INTERPRETER;
9250	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9251	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9252	}
9253
9254
9255	/**
9256	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9257	*/
9258	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9259	{
9260	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9261
9262	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9263	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9264	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9265	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9266	AssertRCReturn(rc, rc);
9267	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9268
9269	PVM pVM = pVCpu->CTX_SUFF(pVM);
9270	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9271	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9272	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9273	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9274
9275	if (RT_LIKELY(rc == VINF_SUCCESS))
9276	{
9277	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9278	Assert(pVmxTransient->cbInstr == 2);
9279	}
9280	return rc;
9281	}
9282
9283
9284	/**
9285	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9286	*/
9287	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9288	{
9289	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9290	PVM pVM = pVCpu->CTX_SUFF(pVM);
9291	int rc = VINF_SUCCESS;
9292
9293	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9294	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9295	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9296	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9297	AssertRCReturn(rc, rc);
9298	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9299
9300	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9301	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9302	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9303
9304	if (RT_LIKELY(rc == VINF_SUCCESS))
9305	{
9306	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9307
9308	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9309	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9310	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9311	{
9312	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9313	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9314	EMInterpretWrmsr() changes it. */
9315	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9316	}
9317	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9318	{
9319	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9320	AssertRCReturn(rc, rc);
9321	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9322	}
9323	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9324	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9325
9326	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9327	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9328	{
9329	switch (pMixedCtx->ecx)
9330	{
9331	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9332	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9333	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9334	case MSR_K8_FS_BASE: /* no break */
9335	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9336	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9337	}
9338	}
9339	#ifdef VBOX_STRICT
9340	else
9341	{
9342	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9343	switch (pMixedCtx->ecx)
9344	{
9345	case MSR_IA32_SYSENTER_CS:
9346	case MSR_IA32_SYSENTER_EIP:
9347	case MSR_IA32_SYSENTER_ESP:
9348	case MSR_K8_FS_BASE:
9349	case MSR_K8_GS_BASE:
9350	{
9351	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9352	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9353	}
9354
9355	case MSR_K8_LSTAR:
9356	case MSR_K6_STAR:
9357	case MSR_K8_SF_MASK:
9358	case MSR_K8_TSC_AUX:
9359	case MSR_K8_KERNEL_GS_BASE:
9360	{
9361	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9362	pMixedCtx->ecx));
9363	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9364	}
9365	}
9366	}
9367	#endif /* VBOX_STRICT */
9368	}
9369	return rc;
9370	}
9371
9372
9373	/**
9374	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9375	*/
9376	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9377	{
9378	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9379
9380	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9382	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9383	return VERR_EM_INTERPRETER;
9384	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9385	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9386	}
9387
9388
9389	/**
9390	* VM-exit handler for when the TPR value is lowered below the specified
9391	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9392	*/
9393	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9394	{
9395	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9396	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9397
9398	/*
9399	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9400	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9401	* resume guest execution.
9402	*/
9403	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9404	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9405	return VINF_SUCCESS;
9406	}
9407
9408
9409	/**
9410	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9411	* VM-exit.
9412	*
9413	* @retval VINF_SUCCESS when guest execution can continue.
9414	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9415	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9416	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9417	* recompiler.
9418	*/
9419	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9420	{
9421	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9422	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9423	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9424	AssertRCReturn(rc, rc);
9425
9426	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9427	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9428	PVM pVM = pVCpu->CTX_SUFF(pVM);
9429	switch (uAccessType)
9430	{
9431	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9432	{
9433	#if 0
9434	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9435	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9436	#else
9437	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9438	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9439	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9440	#endif
9441	AssertRCReturn(rc, rc);
9442
9443	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9444	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9445	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9446	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9447
9448	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9449	{
9450	case 0: /* CR0 */
9451	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9452	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9453	break;
9454	case 2: /* C2 **/
9455	/* Nothing to do here, CR2 it's not part of the VMCS. */
9456	break;
9457	case 3: /* CR3 */
9458	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9459	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9460	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
9461	break;
9462	case 4: /* CR4 */
9463	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9464	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
9465	break;
9466	case 8: /* CR8 */
9467	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9468	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9469	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9470	break;
9471	default:
9472	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9473	break;
9474	}
9475
9476	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9477	break;
9478	}
9479
9480	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9481	{
9482	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9483	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9484	AssertRCReturn(rc, rc);
9485	Assert( !pVM->hm.s.fNestedPaging
9486	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9487	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9488
9489	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9490	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9491	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9492
9493	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9494	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9495	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9496	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9497	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9498	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9499	break;
9500	}
9501
9502	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9503	{
9504	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9505	AssertRCReturn(rc, rc);
9506	rc = EMInterpretCLTS(pVM, pVCpu);
9507	AssertRCReturn(rc, rc);
9508	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9509	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9510	Log4(("CRX CLTS write rc=%d\n", rc));
9511	break;
9512	}
9513
9514	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9515	{
9516	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9517	AssertRCReturn(rc, rc);
9518	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9519	if (RT_LIKELY(rc == VINF_SUCCESS))
9520	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9521	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9522	Log4(("CRX LMSW write rc=%d\n", rc));
9523	break;
9524	}
9525
9526	default:
9527	{
9528	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9529	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9530	}
9531	}
9532
9533	/* Validate possible error codes. */
9534	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9535	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9536	if (RT_SUCCESS(rc))
9537	{
9538	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9539	AssertRCReturn(rc2, rc2);
9540	}
9541
9542	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9543	return rc;
9544	}
9545
9546
9547	/**
9548	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9549	* VM-exit.
9550	*/
9551	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9552	{
9553	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9554	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9555
9556	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9557	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9558	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9559	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9560	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9561	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9562	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9563	AssertRCReturn(rc2, rc2);
9564
9565	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9566	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9567	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9568	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9569	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9570	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9571	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9572
9573	/* I/O operation lookup arrays. */
9574	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9575	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9576
9577	VBOXSTRICTRC rcStrict;
9578	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9579	const uint32_t cbInstr = pVmxTransient->cbInstr;
9580	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9581	PVM pVM = pVCpu->CTX_SUFF(pVM);
9582	if (fIOString)
9583	{
9584	/*
9585	* INS/OUTS - I/O String instruction.
9586	*
9587	* Use instruction-information if available, otherwise fall back on
9588	* interpreting the instruction.
9589	*/
9590	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9591	#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9592	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9593	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9594	{
9595	rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9596	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9597	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9598	AssertRCReturn(rc2, rc2);
9599	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9600	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9601	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9602	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9603	if (fIOWrite)
9604	{
9605	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9606	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9607	//if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9608	// hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9609	// pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9610	}
9611	else
9612	{
9613	AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9614	("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9615	VERR_HMVMX_IPE_4);
9616	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9617	//if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9618	// hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9619	}
9620	}
9621	else
9622	{
9623	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9624	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9625	AssertRCReturn(rc2, rc2);
9626	rcStrict = IEMExecOne(pVCpu);
9627	}
9628	/** @todo IEM needs to be setting these flags somehow. */
9629	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9630	fUpdateRipAlready = true;
9631	#else
9632	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9633	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9634	if (RT_SUCCESS(rcStrict))
9635	{
9636	if (fIOWrite)
9637	{
9638	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9639	(DISCPUMODE)pDis->uAddrMode, cbValue);
9640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9641	}
9642	else
9643	{
9644	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9645	(DISCPUMODE)pDis->uAddrMode, cbValue);
9646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9647	}
9648	}
9649	else
9650	{
9651	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9652	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9653	}
9654	#endif
9655	}
9656	else
9657	{
9658	/*
9659	* IN/OUT - I/O instruction.
9660	*/
9661	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9662	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9663	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9664	if (fIOWrite)
9665	{
9666	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9667	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9668	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9669	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9670	}
9671	else
9672	{
9673	uint32_t u32Result = 0;
9674	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9675	if (IOM_SUCCESS(rcStrict))
9676	{
9677	/* Save result of I/O IN instr. in AL/AX/EAX. */
9678	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9679	}
9680	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9681	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9682	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9683	}
9684	}
9685
9686	if (IOM_SUCCESS(rcStrict))
9687	{
9688	if (!fUpdateRipAlready)
9689	{
9690	pMixedCtx->rip += cbInstr;
9691	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9692	}
9693
9694	/*
9695	* If any I/O breakpoints are armed, we need to check if one triggered
9696	* and take appropriate action.
9697	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9698	*/
9699	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9700	AssertRCReturn(rc2, rc2);
9701
9702	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9703	* execution engines about whether hyper BPs and such are pending. */
9704	uint32_t const uDr7 = pMixedCtx->dr[7];
9705	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9706	&& X86_DR7_ANY_RW_IO(uDr7)
9707	&& (pMixedCtx->cr4 & X86_CR4_DE))
9708	\|\| DBGFBpIsHwIoArmed(pVM)))
9709	{
9710	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9711
9712	/* We're playing with the host CPU state here, make sure we don't preempt. */
9713	HM_DISABLE_PREEMPT_IF_NEEDED();
9714	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9715
9716	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9717	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9718	{
9719	/* Raise #DB. */
9720	if (fIsGuestDbgActive)
9721	ASMSetDR6(pMixedCtx->dr[6]);
9722	if (pMixedCtx->dr[7] != uDr7)
9723	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9724
9725	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9726	}
9727	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9728	else if ( rcStrict2 != VINF_SUCCESS
9729	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9730	rcStrict = rcStrict2;
9731
9732	HM_RESTORE_PREEMPT_IF_NEEDED();
9733	}
9734	}
9735
9736	#ifdef DEBUG
9737	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9738	Assert(!fIOWrite);
9739	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9740	Assert(fIOWrite);
9741	else
9742	{
9743	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9744	* statuses, that the VMM device and some others may return. See
9745	* IOM_SUCCESS() for guidance. */
9746	AssertMsg( RT_FAILURE(rcStrict)
9747	\|\| rcStrict == VINF_SUCCESS
9748	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9749	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9750	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9751	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9752	}
9753	#endif
9754
9755	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9756	return VBOXSTRICTRC_TODO(rcStrict);
9757	}
9758
9759
9760	/**
9761	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9762	* VM-exit.
9763	*/
9764	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9765	{
9766	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9767
9768	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9769	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9770	AssertRCReturn(rc, rc);
9771	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9772	{
9773	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9774	AssertRCReturn(rc, rc);
9775	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9776	{
9777	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9778
9779	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9780	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9781	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9782	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9783	{
9784	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9785	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9786
9787	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9788	Assert(!pVCpu->hm.s.Event.fPending);
9789	pVCpu->hm.s.Event.fPending = true;
9790	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9791	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9792	AssertRCReturn(rc, rc);
9793	if (fErrorCodeValid)
9794	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9795	else
9796	pVCpu->hm.s.Event.u32ErrCode = 0;
9797	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9798	&& uVector == X86_XCPT_PF)
9799	{
9800	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9801	}
9802
9803	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9804	}
9805	}
9806	}
9807
9808	/** @todo Emulate task switch someday, currently just going back to ring-3 for
9809	* emulation. */
9810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9811	return VERR_EM_INTERPRETER;
9812	}
9813
9814
9815	/**
9816	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9817	*/
9818	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9819	{
9820	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9821	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9822	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9823	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9824	AssertRCReturn(rc, rc);
9825	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9826	return VINF_EM_DBG_STEPPED;
9827	}
9828
9829
9830	/**
9831	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9832	*/
9833	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9834	{
9835	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9836
9837	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9838	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9839	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9840	return VINF_SUCCESS;
9841	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9842	return rc;
9843
9844	#if 0
9845	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9846	* just sync the whole thing. */
9847	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9848	#else
9849	/* Aggressive state sync. for now. */
9850	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9851	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9852	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9853	#endif
9854	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9855	AssertRCReturn(rc, rc);
9856
9857	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9858	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9859	switch (uAccessType)
9860	{
9861	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9862	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9863	{
9864	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9865	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9866	{
9867	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9868	}
9869
9870	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9871	GCPhys &= PAGE_BASE_GC_MASK;
9872	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9873	PVM pVM = pVCpu->CTX_SUFF(pVM);
9874	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9875	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9876
9877	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9878	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9879	CPUMCTX2CORE(pMixedCtx), GCPhys);
9880	rc = VBOXSTRICTRC_VAL(rc2);
9881	Log4(("ApicAccess rc=%d\n", rc));
9882	if ( rc == VINF_SUCCESS
9883	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9884	\|\| rc == VERR_PAGE_NOT_PRESENT)
9885	{
9886	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9887	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9888	rc = VINF_SUCCESS;
9889	}
9890	break;
9891	}
9892
9893	default:
9894	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9895	rc = VINF_EM_RAW_EMULATE_INSTR;
9896	break;
9897	}
9898
9899	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9900	return rc;
9901	}
9902
9903
9904	/**
9905	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9906	* VM-exit.
9907	*/
9908	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9909	{
9910	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9911
9912	/* We should -not- get this VM-exit if the guest is debugging. */
9913	if (CPUMIsGuestDebugStateActive(pVCpu))
9914	{
9915	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9916	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9917	}
9918
9919	int rc = VERR_INTERNAL_ERROR_5;
9920	if ( !DBGFIsStepping(pVCpu)
9921	&& !pVCpu->hm.s.fSingleInstruction
9922	&& !CPUMIsHyperDebugStateActive(pVCpu))
9923	{
9924	/* Don't intercept MOV DRx and #DB any more. */
9925	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9926	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9927	AssertRCReturn(rc, rc);
9928
9929	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9930	{
9931	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9932	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9933	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9934	AssertRCReturn(rc, rc);
9935	#endif
9936	}
9937
9938	/* We're playing with the host CPU state here, make sure we can't preempt. */
9939	HM_DISABLE_PREEMPT_IF_NEEDED();
9940
9941	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9942	PVM pVM = pVCpu->CTX_SUFF(pVM);
9943	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9944	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
9945
9946	HM_RESTORE_PREEMPT_IF_NEEDED();
9947
9948	#ifdef VBOX_WITH_STATISTICS
9949	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9950	AssertRCReturn(rc, rc);
9951	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9952	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9953	else
9954	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9955	#endif
9956	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9957	return VINF_SUCCESS;
9958	}
9959
9960	/*
9961	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9962	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9963	*/
9964	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9965	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9966	AssertRCReturn(rc, rc);
9967	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9968
9969	PVM pVM = pVCpu->CTX_SUFF(pVM);
9970	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9971	{
9972	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9973	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9974	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9975	if (RT_SUCCESS(rc))
9976	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9977	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9978	}
9979	else
9980	{
9981	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9982	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9983	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9984	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9985	}
9986
9987	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9988	if (RT_SUCCESS(rc))
9989	{
9990	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9991	AssertRCReturn(rc2, rc2);
9992	}
9993	return rc;
9994	}
9995
9996
9997	/**
9998	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9999	* Conditional VM-exit.
10000	*/
10001	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10002	{
10003	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10004	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10005
10006	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10007	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10008	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10009	return VINF_SUCCESS;
10010	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10011	return rc;
10012
10013	RTGCPHYS GCPhys = 0;
10014	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10015
10016	#if 0
10017	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10018	#else
10019	/* Aggressive state sync. for now. */
10020	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10021	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10022	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10023	#endif
10024	AssertRCReturn(rc, rc);
10025
10026	/*
10027	* If we succeed, resume guest execution.
10028	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10029	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10030	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10031	* weird case. See @bugref{6043}.
10032	*/
10033	PVM pVM = pVCpu->CTX_SUFF(pVM);
10034	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10035	rc = VBOXSTRICTRC_VAL(rc2);
10036	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10037	if ( rc == VINF_SUCCESS
10038	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10039	\|\| rc == VERR_PAGE_NOT_PRESENT)
10040	{
10041	/* Successfully handled MMIO operation. */
10042	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
10043	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
10044	rc = VINF_SUCCESS;
10045	}
10046	return rc;
10047	}
10048
10049
10050	/**
10051	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10052	* VM-exit.
10053	*/
10054	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10055	{
10056	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10057	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10058
10059	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10060	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10061	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10062	return VINF_SUCCESS;
10063	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10064	return rc;
10065
10066	RTGCPHYS GCPhys = 0;
10067	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10068	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10069	#if 0
10070	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10071	#else
10072	/* Aggressive state sync. for now. */
10073	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10074	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10075	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10076	#endif
10077	AssertRCReturn(rc, rc);
10078
10079	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10080	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10081
10082	RTGCUINT uErrorCode = 0;
10083	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10084	uErrorCode \|= X86_TRAP_PF_ID;
10085	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10086	uErrorCode \|= X86_TRAP_PF_RW;
10087	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10088	uErrorCode \|= X86_TRAP_PF_P;
10089
10090	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10091
10092	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10093	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10094
10095	/* Handle the pagefault trap for the nested shadow table. */
10096	PVM pVM = pVCpu->CTX_SUFF(pVM);
10097	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10098	TRPMResetTrap(pVCpu);
10099
10100	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10101	if ( rc == VINF_SUCCESS
10102	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10103	\|\| rc == VERR_PAGE_NOT_PRESENT)
10104	{
10105	/* Successfully synced our nested page tables. */
10106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10107	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
10108	return VINF_SUCCESS;
10109	}
10110
10111	Log4(("EPT return to ring-3 rc=%d\n"));
10112	return rc;
10113	}
10114
10115	/** @} */
10116
10117	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10118	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10119	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10120
10121	/** @name VM-exit exception handlers.
10122	* @{
10123	*/
10124
10125	/**
10126	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10127	*/
10128	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10129	{
10130	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10131	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10132
10133	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10134	AssertRCReturn(rc, rc);
10135
10136	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10137	{
10138	/* Old-style FPU error reporting needs some extra work. */
10139	/** @todo don't fall back to the recompiler, but do it manually. */
10140	return VERR_EM_INTERPRETER;
10141	}
10142
10143	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10144	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10145	return rc;
10146	}
10147
10148
10149	/**
10150	* VM-exit exception handler for #BP (Breakpoint exception).
10151	*/
10152	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10153	{
10154	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10155	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10156
10157	/** @todo Try optimize this by not saving the entire guest state unless
10158	* really needed. */
10159	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10160	AssertRCReturn(rc, rc);
10161
10162	PVM pVM = pVCpu->CTX_SUFF(pVM);
10163	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10164	if (rc == VINF_EM_RAW_GUEST_TRAP)
10165	{
10166	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10167	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10168	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10169	AssertRCReturn(rc, rc);
10170
10171	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10172	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10173	}
10174
10175	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10176	return rc;
10177	}
10178
10179
10180	/**
10181	* VM-exit exception handler for #DB (Debug exception).
10182	*/
10183	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10184	{
10185	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10187	Log6(("XcptDB\n"));
10188
10189	/*
10190	* Get the DR6-like values from the exit qualification and pass it to DBGF
10191	* for processing.
10192	*/
10193	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10194	AssertRCReturn(rc, rc);
10195
10196	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10197	uint64_t uDR6 = X86_DR6_INIT_VAL;
10198	uDR6 \|= ( pVmxTransient->uExitQualification
10199	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10200
10201	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10202	if (rc == VINF_EM_RAW_GUEST_TRAP)
10203	{
10204	/*
10205	* The exception was for the guest. Update DR6, DR7.GD and
10206	* IA32_DEBUGCTL.LBR before forwarding it.
10207	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10208	*/
10209	HM_DISABLE_PREEMPT_IF_NEEDED();
10210
10211	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10212	pMixedCtx->dr[6] \|= uDR6;
10213	if (CPUMIsGuestDebugStateActive(pVCpu))
10214	ASMSetDR6(pMixedCtx->dr[6]);
10215
10216	HM_RESTORE_PREEMPT_IF_NEEDED();
10217
10218	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10219	AssertRCReturn(rc, rc);
10220
10221	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10222	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10223
10224	/* Paranoia. */
10225	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10226	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10227
10228	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10229	AssertRCReturn(rc, rc);
10230
10231	/*
10232	* Raise #DB in the guest.
10233	*/
10234	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10235	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10236	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10237	AssertRCReturn(rc, rc);
10238	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10239	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10240	return VINF_SUCCESS;
10241	}
10242
10243	/*
10244	* Not a guest trap, must be a hypervisor related debug event then.
10245	* Update DR6 in case someone is interested in it.
10246	*/
10247	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10248	AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
10249	CPUMSetHyperDR6(pVCpu, uDR6);
10250
10251	return rc;
10252	}
10253
10254
10255	/**
10256	* VM-exit exception handler for #NM (Device-not-available exception: floating
10257	* point exception).
10258	*/
10259	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10260	{
10261	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10262
10263	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10264	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
10265	#endif
10266
10267	/* We require CR0 and EFER. EFER is always up-to-date. */
10268	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10269	AssertRCReturn(rc, rc);
10270
10271	/* We're playing with the host CPU state here, have to disable preemption. */
10272	HM_DISABLE_PREEMPT_IF_NEEDED();
10273
10274	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10275	PVM pVM = pVCpu->CTX_SUFF(pVM);
10276	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
10277	if (rc == VINF_SUCCESS)
10278	{
10279	Assert(CPUMIsGuestFPUStateActive(pVCpu));
10280	HM_RESTORE_PREEMPT_IF_NEEDED();
10281
10282	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
10283	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10284	return VINF_SUCCESS;
10285	}
10286
10287	HM_RESTORE_PREEMPT_IF_NEEDED();
10288
10289	/* Forward #NM to the guest. */
10290	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10291	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10292	AssertRCReturn(rc, rc);
10293	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10294	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
10295	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10296	return rc;
10297	}
10298
10299
10300	/**
10301	* VM-exit exception handler for #GP (General-protection exception).
10302	*
10303	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10304	*/
10305	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10306	{
10307	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10308	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10309
10310	int rc = VERR_INTERNAL_ERROR_5;
10311	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10312	{
10313	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10314	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10315	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10316	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10317	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10318	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10319	AssertRCReturn(rc, rc);
10320	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10321	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10322	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10323	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10324	return rc;
10325	#else
10326	/* We don't intercept #GP. */
10327	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10328	return VERR_VMX_UNEXPECTED_EXCEPTION;
10329	#endif
10330	}
10331
10332	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10333	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10334
10335	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10336	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10337	AssertRCReturn(rc, rc);
10338
10339	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10340	uint32_t cbOp = 0;
10341	PVM pVM = pVCpu->CTX_SUFF(pVM);
10342	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10343	if (RT_SUCCESS(rc))
10344	{
10345	rc = VINF_SUCCESS;
10346	Assert(cbOp == pDis->cbInstr);
10347	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10348	switch (pDis->pCurInstr->uOpcode)
10349	{
10350	case OP_CLI:
10351	{
10352	pMixedCtx->eflags.Bits.u1IF = 0;
10353	pMixedCtx->rip += pDis->cbInstr;
10354	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
10355	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10356	break;
10357	}
10358
10359	case OP_STI:
10360	{
10361	pMixedCtx->eflags.Bits.u1IF = 1;
10362	pMixedCtx->rip += pDis->cbInstr;
10363	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10364	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10365	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
10366	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10367	break;
10368	}
10369
10370	case OP_HLT:
10371	{
10372	rc = VINF_EM_HALT;
10373	pMixedCtx->rip += pDis->cbInstr;
10374	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
10375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10376	break;
10377	}
10378
10379	case OP_POPF:
10380	{
10381	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10382	uint32_t cbParm = 0;
10383	uint32_t uMask = 0;
10384	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10385	{
10386	cbParm = 4;
10387	uMask = 0xffffffff;
10388	}
10389	else
10390	{
10391	cbParm = 2;
10392	uMask = 0xffff;
10393	}
10394
10395	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10396	RTGCPTR GCPtrStack = 0;
10397	X86EFLAGS Eflags;
10398	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10399	&GCPtrStack);
10400	if (RT_SUCCESS(rc))
10401	{
10402	Assert(sizeof(Eflags.u32) >= cbParm);
10403	Eflags.u32 = 0;
10404	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10405	}
10406	if (RT_FAILURE(rc))
10407	{
10408	rc = VERR_EM_INTERPRETER;
10409	break;
10410	}
10411	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10412	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10413	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10414	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10415	pMixedCtx->eflags.Bits.u1RF = 0;
10416	pMixedCtx->esp += cbParm;
10417	pMixedCtx->esp &= uMask;
10418	pMixedCtx->rip += pDis->cbInstr;
10419	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
10420	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10421	break;
10422	}
10423
10424	case OP_PUSHF:
10425	{
10426	uint32_t cbParm = 0;
10427	uint32_t uMask = 0;
10428	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10429	{
10430	cbParm = 4;
10431	uMask = 0xffffffff;
10432	}
10433	else
10434	{
10435	cbParm = 2;
10436	uMask = 0xffff;
10437	}
10438
10439	/* Get the stack pointer & push the contents of eflags onto the stack. */
10440	RTGCPTR GCPtrStack = 0;
10441	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10442	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10443	if (RT_FAILURE(rc))
10444	{
10445	rc = VERR_EM_INTERPRETER;
10446	break;
10447	}
10448	X86EFLAGS Eflags = pMixedCtx->eflags;
10449	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10450	Eflags.Bits.u1RF = 0;
10451	Eflags.Bits.u1VM = 0;
10452
10453	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10454	if (RT_FAILURE(rc))
10455	{
10456	rc = VERR_EM_INTERPRETER;
10457	break;
10458	}
10459	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10460	pMixedCtx->esp -= cbParm;
10461	pMixedCtx->esp &= uMask;
10462	pMixedCtx->rip += pDis->cbInstr;
10463	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
10464	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10465	break;
10466	}
10467
10468	case OP_IRET:
10469	{
10470	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10471	* instruction reference. */
10472	RTGCPTR GCPtrStack = 0;
10473	uint32_t uMask = 0xffff;
10474	uint16_t aIretFrame[3];
10475	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10476	{
10477	rc = VERR_EM_INTERPRETER;
10478	break;
10479	}
10480	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10481	&GCPtrStack);
10482	if (RT_SUCCESS(rc))
10483	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10484	if (RT_FAILURE(rc))
10485	{
10486	rc = VERR_EM_INTERPRETER;
10487	break;
10488	}
10489	pMixedCtx->eip = 0;
10490	pMixedCtx->ip = aIretFrame[0];
10491	pMixedCtx->cs.Sel = aIretFrame[1];
10492	pMixedCtx->cs.ValidSel = aIretFrame[1];
10493	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10494	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10495	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10496	pMixedCtx->sp += sizeof(aIretFrame);
10497	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
10498	\| HM_CHANGED_GUEST_RFLAGS;
10499	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10500	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10501	break;
10502	}
10503
10504	case OP_INT:
10505	{
10506	uint16_t uVector = pDis->Param1.uValue & 0xff;
10507	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10508	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10509	break;
10510	}
10511
10512	case OP_INTO:
10513	{
10514	if (pMixedCtx->eflags.Bits.u1OF)
10515	{
10516	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10517	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10518	}
10519	break;
10520	}
10521
10522	default:
10523	{
10524	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10525	EMCODETYPE_SUPERVISOR);
10526	rc = VBOXSTRICTRC_VAL(rc2);
10527	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
10528	Log4(("#GP rc=%Rrc\n", rc));
10529	break;
10530	}
10531	}
10532	}
10533	else
10534	rc = VERR_EM_INTERPRETER;
10535
10536	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10537	("#GP Unexpected rc=%Rrc\n", rc));
10538	return rc;
10539	}
10540
10541
10542	/**
10543	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10544	* the exception reported in the VMX transient structure back into the VM.
10545	*
10546	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
10547	* up-to-date.
10548	*/
10549	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10550	{
10551	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10552
10553	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10554	hmR0VmxCheckExitDueToEventDelivery(). */
10555	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10556	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10557	AssertRCReturn(rc, rc);
10558	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10559
10560	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10561	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10562	return VINF_SUCCESS;
10563	}
10564
10565
10566	/**
10567	* VM-exit exception handler for #PF (Page-fault exception).
10568	*/
10569	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10570	{
10571	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10572	PVM pVM = pVCpu->CTX_SUFF(pVM);
10573	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10574	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10575	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10576	AssertRCReturn(rc, rc);
10577
10578	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10579	if (pVM->hm.s.fNestedPaging)
10580	{
10581	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10582	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10583	{
10584	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10585	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10586	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10587	}
10588	else
10589	{
10590	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10591	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10592	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10593	}
10594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10595	return rc;
10596	}
10597	#else
10598	Assert(!pVM->hm.s.fNestedPaging);
10599	#endif
10600
10601	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10602	AssertRCReturn(rc, rc);
10603
10604	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10605	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10606
10607	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10608	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10609	(RTGCPTR)pVmxTransient->uExitQualification);
10610
10611	Log4(("#PF: rc=%Rrc\n", rc));
10612	if (rc == VINF_SUCCESS)
10613	{
10614	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10615	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10616	* memory? We don't update the whole state here... */
10617	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
10618	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
10619	TRPMResetTrap(pVCpu);
10620	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10621	return rc;
10622	}
10623	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10624	{
10625	if (!pVmxTransient->fVectoringPF)
10626	{
10627	/* It's a guest page fault and needs to be reflected to the guest. */
10628	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10629	TRPMResetTrap(pVCpu);
10630	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10631	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10632	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10633	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10634	}
10635	else
10636	{
10637	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10638	TRPMResetTrap(pVCpu);
10639	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10640	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10641	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10642	}
10643
10644	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10645	return VINF_SUCCESS;
10646	}
10647
10648	TRPMResetTrap(pVCpu);
10649	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10650	return rc;
10651	}
10652
10653	/** @} */
10654

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48360

Download in other formats: