HMVMXR0.cpp@ 48749

Last change on this file since 48749 was 48700, checked in by vboxsync, 11 years ago
VMM/HMVMXR0, HMSVMR0: Alignment and assert.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 453.2 KB

Line
1	/* $Id: HMVMXR0.cpp 48700 2013-09-26 07:34:12Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#define HMVMX_ALWAYS_SWAP_FPU_STATE
45	#endif
46
47
48	/*******************************************************************************
49	* Defined Constants And Macros *
50	*******************************************************************************/
51	#if defined(RT_ARCH_AMD64)
52	# define HMVMX_IS_64BIT_HOST_MODE() (true)
53	typedef RTHCUINTREG HMVMXHCUINTREG;
54	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
55	extern "C" uint32_t g_fVMXIs64bitHost;
56	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57	typedef uint64_t HMVMXHCUINTREG;
58	#else
59	# define HMVMX_IS_64BIT_HOST_MODE() (false)
60	typedef RTHCUINTREG HMVMXHCUINTREG;
61	#endif
62
63	/** Use the function table. */
64	#define HMVMX_USE_FUNCTION_TABLE
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	/** @} */
127
128	/** @name
129	* States of the VMCS.
130	*
131	* This does not reflect all possible VMCS states but currently only those
132	* needed for maintaining the VMCS consistently even when thread-context hooks
133	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
134	*/
135	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
136	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
137	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
138	/** @} */
139
140	/**
141	* Exception bitmap mask for real-mode guests (real-on-v86).
142	*
143	* We need to intercept all exceptions manually (except #PF). #NM is also
144	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
145	* even in real-mode if we have Nested Paging support.
146	*/
147	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
148	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
149	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
150	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
151	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
153	\| RT_BIT(X86_XCPT_XF))
154
155	/**
156	* Exception bitmap mask for all contributory exceptions.
157	*
158	* Page fault is deliberately excluded here as it's conditional as to whether
159	* it's contributory or benign. Page faults are handled separately.
160	*/
161	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
162	\| RT_BIT(X86_XCPT_DE))
163
164	/** Maximum VM-instruction error number. */
165	#define HMVMX_INSTR_ERROR_MAX 28
166
167	/** Profiling macro. */
168	#ifdef HM_PROFILE_EXIT_DISPATCH
169	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171	#else
172	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174	#endif
175
176	/** Assert that preemption is disabled or covered by thread-context hooks. */
177	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
178	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180	/** Assert that we haven't migrated CPUs when thread-context hooks are not
181	* used. */
182	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184	("Illegal migration! Entered on CPU %u Current %u\n", \
185	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187	/*******************************************************************************
188	* Structures and Typedefs *
189	*******************************************************************************/
190	/**
191	* VMX transient state.
192	*
193	* A state structure for holding miscellaneous information across
194	* VMX non-root operation and restored after the transition.
195	*/
196	typedef struct VMXTRANSIENT
197	{
198	/** The host's rflags/eflags. */
199	RTCCUINTREG uEflags;
200	#if HC_ARCH_BITS == 32
201	uint32_t u32Alignment0;
202	#endif
203	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
204	uint64_t u64LStarMsr;
205	/** The guest's TPR value used for TPR shadowing. */
206	uint8_t u8GuestTpr;
207	/** Alignment. */
208	uint8_t abAlignment0[7];
209
210	/** The basic VM-exit reason. */
211	uint16_t uExitReason;
212	/** Alignment. */
213	uint16_t u16Alignment0;
214	/** The VM-exit interruption error code. */
215	uint32_t uExitIntrErrorCode;
216	/** The VM-exit exit qualification. */
217	uint64_t uExitQualification;
218
219	/** The VM-exit interruption-information field. */
220	uint32_t uExitIntrInfo;
221	/** The VM-exit instruction-length field. */
222	uint32_t cbInstr;
223	/** The VM-exit instruction-information field. */
224	union
225	{
226	/** Plain unsigned int representation. */
227	uint32_t u;
228	/** INS and OUTS information. */
229	struct
230	{
231	uint32_t u6Reserved0 : 6;
232	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
233	uint32_t u3AddrSize : 3;
234	uint32_t u5Reserved1 : 5;
235	/** The segment register (X86_SREG_XXX). */
236	uint32_t iSegReg : 3;
237	uint32_t uReserved2 : 14;
238	} StrIo;
239	} ExitInstrInfo;
240	/** Whether the VM-entry failed or not. */
241	bool fVMEntryFailed;
242	/** Alignment. */
243	uint8_t abAlignment1[3];
244
245	/** The VM-entry interruption-information field. */
246	uint32_t uEntryIntrInfo;
247	/** The VM-entry exception error code field. */
248	uint32_t uEntryXcptErrorCode;
249	/** The VM-entry instruction length field. */
250	uint32_t cbEntryInstr;
251
252	/** IDT-vectoring information field. */
253	uint32_t uIdtVectoringInfo;
254	/** IDT-vectoring error code. */
255	uint32_t uIdtVectoringErrorCode;
256
257	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
258	uint32_t fVmcsFieldsRead;
259
260	/** Whether the guest FPU was active at the time of VM-exit. */
261	bool fWasGuestFPUStateActive;
262	/** Whether the guest debug state was active at the time of VM-exit. */
263	bool fWasGuestDebugStateActive;
264	/** Whether the hyper debug state was active at the time of VM-exit. */
265	bool fWasHyperDebugStateActive;
266	/** Whether TSC-offsetting should be setup before VM-entry. */
267	bool fUpdateTscOffsettingAndPreemptTimer;
268	/** Whether the VM-exit was caused by a page-fault during delivery of a
269	* contributory exception or a page-fault. */
270	bool fVectoringPF;
271	} VMXTRANSIENT;
272	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
273	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
274	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
275	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
276	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
277	/** Pointer to VMX transient state. */
278	typedef VMXTRANSIENT *PVMXTRANSIENT;
279
280
281	/**
282	* MSR-bitmap read permissions.
283	*/
284	typedef enum VMXMSREXITREAD
285	{
286	/** Reading this MSR causes a VM-exit. */
287	VMXMSREXIT_INTERCEPT_READ = 0xb,
288	/** Reading this MSR does not cause a VM-exit. */
289	VMXMSREXIT_PASSTHRU_READ
290	} VMXMSREXITREAD;
291
292	/**
293	* MSR-bitmap write permissions.
294	*/
295	typedef enum VMXMSREXITWRITE
296	{
297	/** Writing to this MSR causes a VM-exit. */
298	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
299	/** Writing to this MSR does not cause a VM-exit. */
300	VMXMSREXIT_PASSTHRU_WRITE
301	} VMXMSREXITWRITE;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns VBox status code.
307	* @param pVCpu Pointer to the VMCPU.
308	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
309	* out-of-sync. Make sure to update the required
310	* fields before using them.
311	* @param pVmxTransient Pointer to the VMX-transient structure.
312	*/
313	#ifndef HMVMX_USE_FUNCTION_TABLE
314	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315	#else
316	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
317	/** Pointer to VM-exit handler. */
318	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
319	#endif
320
321
322	/*******************************************************************************
323	* Internal Functions *
324	*******************************************************************************/
325	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
326	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
327	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
328	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
329	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
330	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
331	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
332	#endif
333	#ifndef HMVMX_USE_FUNCTION_TABLE
334	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
335	# define HMVMX_EXIT_DECL static int
336	#else
337	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
338	#endif
339
340	/** @name VM-exit handlers.
341	* @{
342	*/
343	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
344	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
345	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
346	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
347	static FNVMXEXITHANDLER hmR0VmxExitSipi;
348	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
349	static FNVMXEXITHANDLER hmR0VmxExitSmi;
350	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
351	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
352	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
353	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
354	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
355	static FNVMXEXITHANDLER hmR0VmxExitHlt;
356	static FNVMXEXITHANDLER hmR0VmxExitInvd;
357	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
358	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
359	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
360	static FNVMXEXITHANDLER hmR0VmxExitRsm;
361	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
362	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
363	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
364	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
365	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
366	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
367	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
368	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
369	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
370	static FNVMXEXITHANDLER hmR0VmxExitMwait;
371	static FNVMXEXITHANDLER hmR0VmxExitMtf;
372	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
373	static FNVMXEXITHANDLER hmR0VmxExitPause;
374	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
375	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
376	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
377	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
378	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
379	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
380	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
381	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
382	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
383	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
384	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
385	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
386	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
387	/** @} */
388
389	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
390	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
391	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
392	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
393	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
394	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
395	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
396	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
397
398	/*******************************************************************************
399	* Global Variables *
400	*******************************************************************************/
401	#ifdef HMVMX_USE_FUNCTION_TABLE
402
403	/**
404	* VMX_EXIT dispatch table.
405	*/
406	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
407	{
408	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
409	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
410	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
411	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
412	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
413	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
414	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
415	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
416	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
417	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
418	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
419	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
420	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
421	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
422	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
423	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
424	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
425	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
426	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
427	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
428	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
429	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
430	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
431	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
432	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
433	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
434	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
435	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
436	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
437	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
438	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
439	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
440	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
441	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
442	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
443	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
444	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
445	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
446	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
447	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
448	/* 40 UNDEFINED */ hmR0VmxExitPause,
449	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
450	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
451	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
452	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
453	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
454	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
455	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
456	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
457	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
458	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
459	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
460	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
461	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
462	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
463	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
464	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
465	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
466	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
467	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
468	};
469	#endif /* HMVMX_USE_FUNCTION_TABLE */
470
471	#ifdef VBOX_STRICT
472	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
473	{
474	/* 0 */ "(Not Used)",
475	/* 1 */ "VMCALL executed in VMX root operation.",
476	/* 2 */ "VMCLEAR with invalid physical address.",
477	/* 3 */ "VMCLEAR with VMXON pointer.",
478	/* 4 */ "VMLAUNCH with non-clear VMCS.",
479	/* 5 */ "VMRESUME with non-launched VMCS.",
480	/* 6 */ "VMRESUME after VMXOFF",
481	/* 7 */ "VM entry with invalid control fields.",
482	/* 8 */ "VM entry with invalid host state fields.",
483	/* 9 */ "VMPTRLD with invalid physical address.",
484	/* 10 */ "VMPTRLD with VMXON pointer.",
485	/* 11 */ "VMPTRLD with incorrect revision identifier.",
486	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
487	/* 13 */ "VMWRITE to read-only VMCS component.",
488	/* 14 */ "(Not Used)",
489	/* 15 */ "VMXON executed in VMX root operation.",
490	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
491	/* 17 */ "VM entry with non-launched executing VMCS.",
492	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
493	/* 19 */ "VMCALL with non-clear VMCS.",
494	/* 20 */ "VMCALL with invalid VM-exit control fields.",
495	/* 21 */ "(Not Used)",
496	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
497	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
498	/* 24 */ "VMCALL with invalid SMM-monitor features.",
499	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
500	/* 26 */ "VM entry with events blocked by MOV SS.",
501	/* 27 */ "(Not Used)",
502	/* 28 */ "Invalid operand to INVEPT/INVVPID."
503	};
504	#endif /* VBOX_STRICT */
505
506
507
508	/**
509	* Updates the VM's last error record. If there was a VMX instruction error,
510	* reads the error data from the VMCS and updates VCPU's last error record as
511	* well.
512	*
513	* @param pVM Pointer to the VM.
514	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
515	* VERR_VMX_UNABLE_TO_START_VM or
516	* VERR_VMX_INVALID_VMCS_FIELD).
517	* @param rc The error code.
518	*/
519	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
520	{
521	AssertPtr(pVM);
522	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
523	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
524	{
525	AssertPtrReturnVoid(pVCpu);
526	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
527	}
528	pVM->hm.s.lLastError = rc;
529	}
530
531
532	/**
533	* Reads the VM-entry interruption-information field from the VMCS into the VMX
534	* transient structure.
535	*
536	* @returns VBox status code.
537	* @param pVmxTransient Pointer to the VMX transient structure.
538	*
539	* @remarks No-long-jump zone!!!
540	*/
541	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
542	{
543	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
544	AssertRCReturn(rc, rc);
545	return VINF_SUCCESS;
546	}
547
548
549	/**
550	* Reads the VM-entry exception error code field from the VMCS into
551	* the VMX transient structure.
552	*
553	* @returns VBox status code.
554	* @param pVmxTransient Pointer to the VMX transient structure.
555	*
556	* @remarks No-long-jump zone!!!
557	*/
558	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
559	{
560	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
561	AssertRCReturn(rc, rc);
562	return VINF_SUCCESS;
563	}
564
565
566	/**
567	* Reads the VM-entry exception error code field from the VMCS into
568	* the VMX transient structure.
569	*
570	* @returns VBox status code.
571	* @param pVCpu Pointer to the VMCPU.
572	* @param pVmxTransient Pointer to the VMX transient structure.
573	*
574	* @remarks No-long-jump zone!!!
575	*/
576	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
577	{
578	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
579	AssertRCReturn(rc, rc);
580	return VINF_SUCCESS;
581	}
582
583
584	/**
585	* Reads the VM-exit interruption-information field from the VMCS into the VMX
586	* transient structure.
587	*
588	* @returns VBox status code.
589	* @param pVCpu Pointer to the VMCPU.
590	* @param pVmxTransient Pointer to the VMX transient structure.
591	*/
592	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
593	{
594	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
595	{
596	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
597	AssertRCReturn(rc, rc);
598	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
599	}
600	return VINF_SUCCESS;
601	}
602
603
604	/**
605	* Reads the VM-exit interruption error code from the VMCS into the VMX
606	* transient structure.
607	*
608	* @returns VBox status code.
609	* @param pVCpu Pointer to the VMCPU.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*/
612	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
613	{
614	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
615	{
616	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
617	AssertRCReturn(rc, rc);
618	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
619	}
620	return VINF_SUCCESS;
621	}
622
623
624	/**
625	* Reads the VM-exit instruction length field from the VMCS into the VMX
626	* transient structure.
627	*
628	* @returns VBox status code.
629	* @param pVCpu Pointer to the VMCPU.
630	* @param pVmxTransient Pointer to the VMX transient structure.
631	*/
632	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
633	{
634	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
635	{
636	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
637	AssertRCReturn(rc, rc);
638	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
639	}
640	return VINF_SUCCESS;
641	}
642
643
644	/**
645	* Reads the VM-exit instruction-information field from the VMCS into
646	* the VMX transient structure.
647	*
648	* @returns VBox status code.
649	* @param pVCpu The cross context per CPU structure.
650	* @param pVmxTransient Pointer to the VMX transient structure.
651	*/
652	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
653	{
654	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
655	{
656	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
657	AssertRCReturn(rc, rc);
658	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
659	}
660	return VINF_SUCCESS;
661	}
662
663
664	/**
665	* Reads the exit qualification from the VMCS into the VMX transient structure.
666	*
667	* @returns VBox status code.
668	* @param pVCpu Pointer to the VMCPU.
669	* @param pVmxTransient Pointer to the VMX transient structure.
670	*/
671	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
672	{
673	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
674	{
675	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
676	AssertRCReturn(rc, rc);
677	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
678	}
679	return VINF_SUCCESS;
680	}
681
682
683	/**
684	* Reads the IDT-vectoring information field from the VMCS into the VMX
685	* transient structure.
686	*
687	* @returns VBox status code.
688	* @param pVmxTransient Pointer to the VMX transient structure.
689	*
690	* @remarks No-long-jump zone!!!
691	*/
692	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
693	{
694	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
695	{
696	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
697	AssertRCReturn(rc, rc);
698	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
699	}
700	return VINF_SUCCESS;
701	}
702
703
704	/**
705	* Reads the IDT-vectoring error code from the VMCS into the VMX
706	* transient structure.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient Pointer to the VMX transient structure.
710	*/
711	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
712	{
713	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
714	{
715	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
716	AssertRCReturn(rc, rc);
717	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
718	}
719	return VINF_SUCCESS;
720	}
721
722
723	/**
724	* Enters VMX root mode operation on the current CPU.
725	*
726	* @returns VBox status code.
727	* @param pVM Pointer to the VM (optional, can be NULL, after
728	* a resume).
729	* @param HCPhysCpuPage Physical address of the VMXON region.
730	* @param pvCpuPage Pointer to the VMXON region.
731	*/
732	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
733	{
734	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
735	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
736	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
737
738	if (pVM)
739	{
740	/* Write the VMCS revision dword to the VMXON region. */
741	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
742	}
743
744	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
745	RTCCUINTREG uEflags = ASMIntDisableFlags();
746
747	/* Enable the VMX bit in CR4 if necessary. */
748	RTCCUINTREG uCr4 = ASMGetCR4();
749	if (!(uCr4 & X86_CR4_VMXE))
750	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
751
752	/* Enter VMX root mode. */
753	int rc = VMXEnable(HCPhysCpuPage);
754	if (RT_FAILURE(rc))
755	ASMSetCR4(uCr4);
756
757	/* Restore interrupts. */
758	ASMSetFlags(uEflags);
759	return rc;
760	}
761
762
763	/**
764	* Exits VMX root mode operation on the current CPU.
765	*
766	* @returns VBox status code.
767	*/
768	static int hmR0VmxLeaveRootMode(void)
769	{
770	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
771
772	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
773	RTCCUINTREG uEflags = ASMIntDisableFlags();
774
775	/* If we're for some reason not in VMX root mode, then don't leave it. */
776	RTCCUINTREG uHostCR4 = ASMGetCR4();
777
778	int rc;
779	if (uHostCR4 & X86_CR4_VMXE)
780	{
781	/* Exit VMX root mode and clear the VMX bit in CR4. */
782	VMXDisable();
783	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
784	rc = VINF_SUCCESS;
785	}
786	else
787	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
788
789	/* Restore interrupts. */
790	ASMSetFlags(uEflags);
791	return rc;
792	}
793
794
795	/**
796	* Allocates and maps one physically contiguous page. The allocated page is
797	* zero'd out. (Used by various VT-x structures).
798	*
799	* @returns IPRT status code.
800	* @param pMemObj Pointer to the ring-0 memory object.
801	* @param ppVirt Where to store the virtual address of the
802	* allocation.
803	* @param pPhys Where to store the physical address of the
804	* allocation.
805	*/
806	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
807	{
808	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
809	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
810	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
811
812	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
813	if (RT_FAILURE(rc))
814	return rc;
815	ppVirt = RTR0MemObjAddress(pMemObj);
816	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
817	ASMMemZero32(*ppVirt, PAGE_SIZE);
818	return VINF_SUCCESS;
819	}
820
821
822	/**
823	* Frees and unmaps an allocated physical page.
824	*
825	* @param pMemObj Pointer to the ring-0 memory object.
826	* @param ppVirt Where to re-initialize the virtual address of
827	* allocation as 0.
828	* @param pHCPhys Where to re-initialize the physical address of the
829	* allocation as 0.
830	*/
831	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
832	{
833	AssertPtr(pMemObj);
834	AssertPtr(ppVirt);
835	AssertPtr(pHCPhys);
836	if (*pMemObj != NIL_RTR0MEMOBJ)
837	{
838	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
839	AssertRC(rc);
840	*pMemObj = NIL_RTR0MEMOBJ;
841	*ppVirt = 0;
842	*pHCPhys = 0;
843	}
844	}
845
846
847	/**
848	* Worker function to free VT-x related structures.
849	*
850	* @returns IPRT status code.
851	* @param pVM Pointer to the VM.
852	*/
853	static void hmR0VmxStructsFree(PVM pVM)
854	{
855	for (VMCPUID i = 0; i < pVM->cCpus; i++)
856	{
857	PVMCPU pVCpu = &pVM->aCpus[i];
858	AssertPtr(pVCpu);
859
860	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
861	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
862	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
863	#endif
864
865	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
866	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
867
868	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
869	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
870	}
871
872	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
873	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
874	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
875	#endif
876	}
877
878
879	/**
880	* Worker function to allocate VT-x related VM structures.
881	*
882	* @returns IPRT status code.
883	* @param pVM Pointer to the VM.
884	*/
885	static int hmR0VmxStructsAlloc(PVM pVM)
886	{
887	/*
888	* Initialize members up-front so we can cleanup properly on allocation failure.
889	*/
890	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
891	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
892	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
893	pVM->hm.s.vmx.HCPhys##a_Name = 0;
894
895	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
896	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
897	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
898	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
899
900	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
901	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
902	#endif
903	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
904
905	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
906	for (VMCPUID i = 0; i < pVM->cCpus; i++)
907	{
908	PVMCPU pVCpu = &pVM->aCpus[i];
909	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
910	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
911	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
912	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
913	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
914	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
915	#endif
916	}
917	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
918	#undef VMXLOCAL_INIT_VM_MEMOBJ
919
920	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
921	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
922	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
923	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
924
925	/*
926	* Allocate all the VT-x structures.
927	*/
928	int rc = VINF_SUCCESS;
929	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
930	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
931	if (RT_FAILURE(rc))
932	goto cleanup;
933	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
934	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
935	#endif
936
937	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
938	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
939	{
940	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
941	&pVM->hm.s.vmx.HCPhysApicAccess);
942	if (RT_FAILURE(rc))
943	goto cleanup;
944	}
945
946	/*
947	* Initialize per-VCPU VT-x structures.
948	*/
949	for (VMCPUID i = 0; i < pVM->cCpus; i++)
950	{
951	PVMCPU pVCpu = &pVM->aCpus[i];
952	AssertPtr(pVCpu);
953
954	/* Allocate the VM control structure (VMCS). */
955	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
956	if (RT_FAILURE(rc))
957	goto cleanup;
958
959	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
960	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
961	{
962	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
963	&pVCpu->hm.s.vmx.HCPhysVirtApic);
964	if (RT_FAILURE(rc))
965	goto cleanup;
966	}
967
968	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
969	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
970	{
971	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
972	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
973	if (RT_FAILURE(rc))
974	goto cleanup;
975	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
976	}
977
978	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
979	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
980	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
981	if (RT_FAILURE(rc))
982	goto cleanup;
983
984	/* Allocate the VM-exit MSR-load page for the host MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988	#endif
989	}
990
991	return VINF_SUCCESS;
992
993	cleanup:
994	hmR0VmxStructsFree(pVM);
995	return rc;
996	}
997
998
999	/**
1000	* Does global VT-x initialization (called during module initialization).
1001	*
1002	* @returns VBox status code.
1003	*/
1004	VMMR0DECL(int) VMXR0GlobalInit(void)
1005	{
1006	#ifdef HMVMX_USE_FUNCTION_TABLE
1007	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1008	# ifdef VBOX_STRICT
1009	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1010	Assert(g_apfnVMExitHandlers[i]);
1011	# endif
1012	#endif
1013	return VINF_SUCCESS;
1014	}
1015
1016
1017	/**
1018	* Does global VT-x termination (called during module termination).
1019	*/
1020	VMMR0DECL(void) VMXR0GlobalTerm()
1021	{
1022	/* Nothing to do currently. */
1023	}
1024
1025
1026	/**
1027	* Sets up and activates VT-x on the current CPU.
1028	*
1029	* @returns VBox status code.
1030	* @param pCpu Pointer to the global CPU info struct.
1031	* @param pVM Pointer to the VM (can be NULL after a host resume
1032	* operation).
1033	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1034	* fEnabledByHost is true).
1035	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1036	* @a fEnabledByHost is true).
1037	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1038	* enable VT-x on the host.
1039	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1040	*/
1041	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1042	void *pvMsrs)
1043	{
1044	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1045	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1046	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1047
1048	/* Enable VT-x if it's not already enabled by the host. */
1049	if (!fEnabledByHost)
1050	{
1051	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1052	if (RT_FAILURE(rc))
1053	return rc;
1054	}
1055
1056	/*
1057	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1058	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1059	*/
1060	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1061	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1062	{
1063	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1064	pCpu->fFlushAsidBeforeUse = false;
1065	}
1066	else
1067	pCpu->fFlushAsidBeforeUse = true;
1068
1069	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1070	++pCpu->cTlbFlushes;
1071
1072	return VINF_SUCCESS;
1073	}
1074
1075
1076	/**
1077	* Deactivates VT-x on the current CPU.
1078	*
1079	* @returns VBox status code.
1080	* @param pCpu Pointer to the global CPU info struct.
1081	* @param pvCpuPage Pointer to the VMXON region.
1082	* @param HCPhysCpuPage Physical address of the VMXON region.
1083	*
1084	* @remarks This function should never be called when SUPR0EnableVTx() or
1085	* similar was used to enable VT-x on the host.
1086	*/
1087	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1088	{
1089	NOREF(pCpu);
1090	NOREF(pvCpuPage);
1091	NOREF(HCPhysCpuPage);
1092
1093	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1094	return hmR0VmxLeaveRootMode();
1095	}
1096
1097
1098	/**
1099	* Sets the permission bits for the specified MSR in the MSR bitmap.
1100	*
1101	* @param pVCpu Pointer to the VMCPU.
1102	* @param uMSR The MSR value.
1103	* @param enmRead Whether reading this MSR causes a VM-exit.
1104	* @param enmWrite Whether writing this MSR causes a VM-exit.
1105	*/
1106	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1107	{
1108	int32_t iBit;
1109	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1110
1111	/*
1112	* Layout:
1113	* 0x000 - 0x3ff - Low MSR read bits
1114	* 0x400 - 0x7ff - High MSR read bits
1115	* 0x800 - 0xbff - Low MSR write bits
1116	* 0xc00 - 0xfff - High MSR write bits
1117	*/
1118	if (uMsr <= 0x00001FFF)
1119	iBit = uMsr;
1120	else if ( uMsr >= 0xC0000000
1121	&& uMsr <= 0xC0001FFF)
1122	{
1123	iBit = (uMsr - 0xC0000000);
1124	pbMsrBitmap += 0x400;
1125	}
1126	else
1127	{
1128	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1129	return;
1130	}
1131
1132	Assert(iBit <= 0x1fff);
1133	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1134	ASMBitSet(pbMsrBitmap, iBit);
1135	else
1136	ASMBitClear(pbMsrBitmap, iBit);
1137
1138	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1139	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1140	else
1141	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1142	}
1143
1144
1145	/**
1146	* Flushes the TLB using EPT.
1147	*
1148	* @returns VBox status code.
1149	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1150	* enmFlush).
1151	* @param enmFlush Type of flush.
1152	*
1153	* @remarks Caller is responsible for making sure this function is called only
1154	* when NestedPaging is supported and providing @a enmFlush that is
1155	* supported by the CPU.
1156	*/
1157	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1158	{
1159	uint64_t au64Descriptor[2];
1160	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1161	au64Descriptor[0] = 0;
1162	else
1163	{
1164	Assert(pVCpu);
1165	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1166	}
1167	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1168
1169	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1170	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1171	rc));
1172	if ( RT_SUCCESS(rc)
1173	&& pVCpu)
1174	{
1175	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1176	}
1177	}
1178
1179
1180	/**
1181	* Flushes the TLB using VPID.
1182	*
1183	* @returns VBox status code.
1184	* @param pVM Pointer to the VM.
1185	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1186	* enmFlush).
1187	* @param enmFlush Type of flush.
1188	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1189	* on @a enmFlush).
1190	*/
1191	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1192	{
1193	AssertPtr(pVM);
1194	Assert(pVM->hm.s.vmx.fVpid);
1195
1196	uint64_t au64Descriptor[2];
1197	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1198	{
1199	au64Descriptor[0] = 0;
1200	au64Descriptor[1] = 0;
1201	}
1202	else
1203	{
1204	AssertPtr(pVCpu);
1205	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1206	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1207	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1208	au64Descriptor[1] = GCPtr;
1209	}
1210
1211	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1212	AssertMsg(rc == VINF_SUCCESS,
1213	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1214	if ( RT_SUCCESS(rc)
1215	&& pVCpu)
1216	{
1217	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1218	}
1219	}
1220
1221
1222	/**
1223	* Invalidates a guest page by guest virtual address. Only relevant for
1224	* EPT/VPID, otherwise there is nothing really to invalidate.
1225	*
1226	* @returns VBox status code.
1227	* @param pVM Pointer to the VM.
1228	* @param pVCpu Pointer to the VMCPU.
1229	* @param GCVirt Guest virtual address of the page to invalidate.
1230	*/
1231	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1232	{
1233	AssertPtr(pVM);
1234	AssertPtr(pVCpu);
1235	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1236
1237	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1238	if (!fFlushPending)
1239	{
1240	/*
1241	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1242	* See @bugref{6043} and @bugref{6177}.
1243	*
1244	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1245	* function maybe called in a loop with individual addresses.
1246	*/
1247	if (pVM->hm.s.vmx.fVpid)
1248	{
1249	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1250	{
1251	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1252	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1253	}
1254	else
1255	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1256	}
1257	else if (pVM->hm.s.fNestedPaging)
1258	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1259	}
1260
1261	return VINF_SUCCESS;
1262	}
1263
1264
1265	/**
1266	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1267	* otherwise there is nothing really to invalidate.
1268	*
1269	* @returns VBox status code.
1270	* @param pVM Pointer to the VM.
1271	* @param pVCpu Pointer to the VMCPU.
1272	* @param GCPhys Guest physical address of the page to invalidate.
1273	*/
1274	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1275	{
1276	LogFlowFunc(("%RGp\n", GCPhys));
1277
1278	/*
1279	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1280	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1281	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1282	*/
1283	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1284	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1285	return VINF_SUCCESS;
1286	}
1287
1288
1289	/**
1290	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1291	* case where neither EPT nor VPID is supported by the CPU.
1292	*
1293	* @param pVM Pointer to the VM.
1294	* @param pVCpu Pointer to the VMCPU.
1295	* @param pCpu Pointer to the global HM struct.
1296	*
1297	* @remarks Called with interrupts disabled.
1298	*/
1299	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1300	{
1301	AssertPtr(pVCpu);
1302	AssertPtr(pCpu);
1303	NOREF(pVM);
1304
1305	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1306	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1307
1308	pVCpu->hm.s.TlbShootdown.cPages = 0;
1309	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1310	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1311	pVCpu->hm.s.fForceTLBFlush = false;
1312	return;
1313	}
1314
1315
1316	/**
1317	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1318	*
1319	* @param pVM Pointer to the VM.
1320	* @param pVCpu Pointer to the VMCPU.
1321	* @param pCpu Pointer to the global HM CPU struct.
1322	* @remarks All references to "ASID" in this function pertains to "VPID" in
1323	* Intel's nomenclature. The reason is, to avoid confusion in compare
1324	* statements since the host-CPU copies are named "ASID".
1325	*
1326	* @remarks Called with interrupts disabled.
1327	*/
1328	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1329	{
1330	#ifdef VBOX_WITH_STATISTICS
1331	bool fTlbFlushed = false;
1332	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1333	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1334	if (!fTlbFlushed) \
1335	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1336	} while (0)
1337	#else
1338	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1339	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1340	#endif
1341
1342	AssertPtr(pVM);
1343	AssertPtr(pCpu);
1344	AssertPtr(pVCpu);
1345	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1346	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1347	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1348
1349
1350	/*
1351	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1352	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1353	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1354	*/
1355	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1356	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1357	{
1358	++pCpu->uCurrentAsid;
1359	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1360	{
1361	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1362	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1363	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1364	}
1365
1366	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1367	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1368	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1369
1370	/*
1371	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1372	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1373	*/
1374	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1375	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1376	HMVMX_SET_TAGGED_TLB_FLUSHED();
1377	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1378	}
1379
1380	/* Check for explicit TLB shootdowns. */
1381	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1382	{
1383	/*
1384	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1385	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1386	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1387	* but not guest-physical mappings.
1388	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1389	*/
1390	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1391	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1392	HMVMX_SET_TAGGED_TLB_FLUSHED();
1393	}
1394
1395	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1396	* not be executed. See hmQueueInvlPage() where it is commented
1397	* out. Support individual entry flushing someday. */
1398	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1399	{
1400	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1401
1402	/*
1403	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1404	* as supported by the CPU.
1405	*/
1406	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1407	{
1408	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1409	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1410	}
1411	else
1412	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1413
1414	HMVMX_SET_TAGGED_TLB_FLUSHED();
1415	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1416	}
1417
1418	pVCpu->hm.s.TlbShootdown.cPages = 0;
1419	pVCpu->hm.s.fForceTLBFlush = false;
1420
1421	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1422
1423	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1424	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1425	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1426	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1427	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1428	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1429	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1430	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1431
1432	/* Update VMCS with the VPID. */
1433	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1434	AssertRC(rc);
1435
1436	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1437	}
1438
1439
1440	/**
1441	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1442	*
1443	* @returns VBox status code.
1444	* @param pVM Pointer to the VM.
1445	* @param pVCpu Pointer to the VMCPU.
1446	* @param pCpu Pointer to the global HM CPU struct.
1447	*
1448	* @remarks Called with interrupts disabled.
1449	*/
1450	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1451	{
1452	AssertPtr(pVM);
1453	AssertPtr(pVCpu);
1454	AssertPtr(pCpu);
1455	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1456	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1457
1458	/*
1459	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1460	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1461	*/
1462	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1463	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1464	{
1465	pVCpu->hm.s.fForceTLBFlush = true;
1466	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1467	}
1468
1469	/* Check for explicit TLB shootdown flushes. */
1470	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1471	{
1472	pVCpu->hm.s.fForceTLBFlush = true;
1473	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1474	}
1475
1476	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1477	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1478
1479	if (pVCpu->hm.s.fForceTLBFlush)
1480	{
1481	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1482	pVCpu->hm.s.fForceTLBFlush = false;
1483	}
1484	else
1485	{
1486	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1487	* not be executed. See hmQueueInvlPage() where it is commented
1488	* out. Support individual entry flushing someday. */
1489	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1490	{
1491	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1492	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1493	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1494	}
1495	else
1496	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1497	}
1498
1499	pVCpu->hm.s.TlbShootdown.cPages = 0;
1500	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1501	}
1502
1503
1504	/**
1505	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1506	*
1507	* @returns VBox status code.
1508	* @param pVM Pointer to the VM.
1509	* @param pVCpu Pointer to the VMCPU.
1510	* @param pCpu Pointer to the global HM CPU struct.
1511	*
1512	* @remarks Called with interrupts disabled.
1513	*/
1514	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1515	{
1516	AssertPtr(pVM);
1517	AssertPtr(pVCpu);
1518	AssertPtr(pCpu);
1519	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1520	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1521
1522	/*
1523	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1524	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1525	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1526	*/
1527	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1528	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1529	{
1530	pVCpu->hm.s.fForceTLBFlush = true;
1531	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1532	}
1533
1534	/* Check for explicit TLB shootdown flushes. */
1535	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1536	{
1537	/*
1538	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1539	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1540	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1541	*/
1542	pVCpu->hm.s.fForceTLBFlush = true;
1543	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1544	}
1545
1546	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1547	if (pVCpu->hm.s.fForceTLBFlush)
1548	{
1549	++pCpu->uCurrentAsid;
1550	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1551	{
1552	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1553	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1554	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1555	}
1556
1557	pVCpu->hm.s.fForceTLBFlush = false;
1558	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1559	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1560	if (pCpu->fFlushAsidBeforeUse)
1561	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1562	}
1563	else
1564	{
1565	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1566	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1567	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1568	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1569
1570	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1571	* not be executed. See hmQueueInvlPage() where it is commented
1572	* out. Support individual entry flushing someday. */
1573	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1574	{
1575	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1576	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1577	{
1578	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1579	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1580	}
1581	else
1582	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1583	}
1584	else
1585	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1586	}
1587
1588	pVCpu->hm.s.TlbShootdown.cPages = 0;
1589	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1590
1591	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1592	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1593	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1594	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1595	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1596	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1597
1598	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1599	AssertRC(rc);
1600	}
1601
1602
1603	/**
1604	* Flushes the guest TLB entry based on CPU capabilities.
1605	*
1606	* @param pVCpu Pointer to the VMCPU.
1607	* @param pCpu Pointer to the global HM CPU struct.
1608	*/
1609	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1610	{
1611	PVM pVM = pVCpu->CTX_SUFF(pVM);
1612	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1613	{
1614	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1615	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1616	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1617	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1618	default:
1619	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1620	break;
1621	}
1622	}
1623
1624
1625	/**
1626	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1627	* TLB entries from the host TLB before VM-entry.
1628	*
1629	* @returns VBox status code.
1630	* @param pVM Pointer to the VM.
1631	*/
1632	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1633	{
1634	/*
1635	* Determine optimal flush type for Nested Paging.
1636	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1637	* guest execution (see hmR3InitFinalizeR0()).
1638	*/
1639	if (pVM->hm.s.fNestedPaging)
1640	{
1641	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1642	{
1643	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1644	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1645	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1646	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1647	else
1648	{
1649	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1650	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1651	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1652	}
1653
1654	/* Make sure the write-back cacheable memory type for EPT is supported. */
1655	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1656	{
1657	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1658	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1659	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1660	}
1661	}
1662	else
1663	{
1664	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1665	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1666	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1667	}
1668	}
1669
1670	/*
1671	* Determine optimal flush type for VPID.
1672	*/
1673	if (pVM->hm.s.vmx.fVpid)
1674	{
1675	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1676	{
1677	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1678	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1679	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1680	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1681	else
1682	{
1683	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1684	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1685	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1686	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1687	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1688	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1689	pVM->hm.s.vmx.fVpid = false;
1690	}
1691	}
1692	else
1693	{
1694	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1695	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1696	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1697	pVM->hm.s.vmx.fVpid = false;
1698	}
1699	}
1700
1701	/*
1702	* Setup the handler for flushing tagged-TLBs.
1703	*/
1704	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1705	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1706	else if (pVM->hm.s.fNestedPaging)
1707	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1708	else if (pVM->hm.s.vmx.fVpid)
1709	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1710	else
1711	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1712	return VINF_SUCCESS;
1713	}
1714
1715
1716	/**
1717	* Sets up pin-based VM-execution controls in the VMCS.
1718	*
1719	* @returns VBox status code.
1720	* @param pVM Pointer to the VM.
1721	* @param pVCpu Pointer to the VMCPU.
1722	*/
1723	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1724	{
1725	AssertPtr(pVM);
1726	AssertPtr(pVCpu);
1727
1728	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1729	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1730
1731	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1732	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1733	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1734
1735	/* Enable the VMX preemption timer. */
1736	if (pVM->hm.s.vmx.fUsePreemptTimer)
1737	{
1738	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1739	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1740	}
1741
1742	if ((val & zap) != val)
1743	{
1744	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1745	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1746	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1747	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1748	}
1749
1750	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1751	AssertRCReturn(rc, rc);
1752
1753	/* Update VCPU with the currently set pin-based VM-execution controls. */
1754	pVCpu->hm.s.vmx.u32PinCtls = val;
1755	return rc;
1756	}
1757
1758
1759	/**
1760	* Sets up processor-based VM-execution controls in the VMCS.
1761	*
1762	* @returns VBox status code.
1763	* @param pVM Pointer to the VM.
1764	* @param pVMCPU Pointer to the VMCPU.
1765	*/
1766	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1767	{
1768	AssertPtr(pVM);
1769	AssertPtr(pVCpu);
1770
1771	int rc = VERR_INTERNAL_ERROR_5;
1772	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1773	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1774
1775	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1776	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1777	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1778	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1779	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1780	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1781	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1782
1783	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1784	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1785	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1786	{
1787	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1788	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1789	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1790	}
1791
1792	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1793	if (!pVM->hm.s.fNestedPaging)
1794	{
1795	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1796	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1797	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1798	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1799	}
1800
1801	/* Use TPR shadowing if supported by the CPU. */
1802	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1803	{
1804	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1805	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1806	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1807	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1808	AssertRCReturn(rc, rc);
1809
1810	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1811	/* CR8 writes causes a VM-exit based on TPR threshold. */
1812	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1813	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1814	}
1815	else
1816	{
1817	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1818	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1819	}
1820
1821	/* Use MSR-bitmaps if supported by the CPU. */
1822	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1823	{
1824	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1825
1826	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1827	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1828	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1829	AssertRCReturn(rc, rc);
1830
1831	/*
1832	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1833	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1834	*/
1835	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1836	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1837	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1838	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1839	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1840	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1841	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1842	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1843	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1844	}
1845
1846	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1847	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1848	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1849
1850	if ((val & zap) != val)
1851	{
1852	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1853	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1854	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1855	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1856	}
1857
1858	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1859	AssertRCReturn(rc, rc);
1860
1861	/* Update VCPU with the currently set processor-based VM-execution controls. */
1862	pVCpu->hm.s.vmx.u32ProcCtls = val;
1863
1864	/*
1865	* Secondary processor-based VM-execution controls.
1866	*/
1867	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1868	{
1869	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1870	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1871
1872	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1873	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1874
1875	if (pVM->hm.s.fNestedPaging)
1876	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1877	else
1878	{
1879	/*
1880	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1881	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1882	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1883	*/
1884	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1885	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1886	}
1887
1888	if (pVM->hm.s.vmx.fVpid)
1889	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1890
1891	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1892	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1893
1894	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1895	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1896	* done dynamically. */
1897	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1898	{
1899	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1900	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1901	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1902	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1903	AssertRCReturn(rc, rc);
1904	}
1905
1906	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1907	{
1908	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1909	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1910	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1911	}
1912
1913	if ((val & zap) != val)
1914	{
1915	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1916	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1917	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1918	}
1919
1920	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1921	AssertRCReturn(rc, rc);
1922
1923	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1924	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1925	}
1926	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1927	{
1928	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1929	"available\n"));
1930	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1931	}
1932
1933	return VINF_SUCCESS;
1934	}
1935
1936
1937	/**
1938	* Sets up miscellaneous (everything other than Pin & Processor-based
1939	* VM-execution) control fields in the VMCS.
1940	*
1941	* @returns VBox status code.
1942	* @param pVM Pointer to the VM.
1943	* @param pVCpu Pointer to the VMCPU.
1944	*/
1945	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1946	{
1947	AssertPtr(pVM);
1948	AssertPtr(pVCpu);
1949
1950	int rc = VERR_GENERAL_FAILURE;
1951
1952	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1953	#if 0
1954	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
1955	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1956	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1957
1958	/*
1959	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1960	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1961	* We thus use the exception bitmap to control it rather than use both.
1962	*/
1963	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1964	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1965
1966	/** @todo Explore possibility of using IO-bitmaps. */
1967	/* All IO & IOIO instructions cause VM-exits. */
1968	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1969	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1970
1971	/* Initialize the MSR-bitmap area. */
1972	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1973	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1974	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1975	#endif
1976
1977	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1978	/* Setup MSR autoloading/storing. */
1979	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1980	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1981	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1982	AssertRCReturn(rc, rc);
1983	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1984	AssertRCReturn(rc, rc);
1985
1986	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1987	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1988	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1989	AssertRCReturn(rc, rc);
1990	#endif
1991
1992	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1993	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1994	AssertRCReturn(rc, rc);
1995
1996	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1997	#if 0
1998	/* Setup debug controls */
1999	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2000	AssertRCReturn(rc, rc);
2001	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2002	AssertRCReturn(rc, rc);
2003	#endif
2004
2005	return rc;
2006	}
2007
2008
2009	/**
2010	* Sets up the initial exception bitmap in the VMCS based on static conditions
2011	* (i.e. conditions that cannot ever change after starting the VM).
2012	*
2013	* @returns VBox status code.
2014	* @param pVM Pointer to the VM.
2015	* @param pVCpu Pointer to the VMCPU.
2016	*/
2017	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2018	{
2019	AssertPtr(pVM);
2020	AssertPtr(pVCpu);
2021
2022	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2023
2024	uint32_t u32XcptBitmap = 0;
2025
2026	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2027	if (!pVM->hm.s.fNestedPaging)
2028	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2029
2030	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2031	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2032	AssertRCReturn(rc, rc);
2033	return rc;
2034	}
2035
2036
2037	/**
2038	* Sets up the initial guest-state mask. The guest-state mask is consulted
2039	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2040	* for the nested virtualization case (as it would cause a VM-exit).
2041	*
2042	* @param pVCpu Pointer to the VMCPU.
2043	*/
2044	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2045	{
2046	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2047	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2048	return VINF_SUCCESS;
2049	}
2050
2051
2052	/**
2053	* Does per-VM VT-x initialization.
2054	*
2055	* @returns VBox status code.
2056	* @param pVM Pointer to the VM.
2057	*/
2058	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2059	{
2060	LogFlowFunc(("pVM=%p\n", pVM));
2061
2062	int rc = hmR0VmxStructsAlloc(pVM);
2063	if (RT_FAILURE(rc))
2064	{
2065	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2066	return rc;
2067	}
2068
2069	return VINF_SUCCESS;
2070	}
2071
2072
2073	/**
2074	* Does per-VM VT-x termination.
2075	*
2076	* @returns VBox status code.
2077	* @param pVM Pointer to the VM.
2078	*/
2079	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2080	{
2081	LogFlowFunc(("pVM=%p\n", pVM));
2082
2083	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2084	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2085	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2086	#endif
2087	hmR0VmxStructsFree(pVM);
2088	return VINF_SUCCESS;
2089	}
2090
2091
2092	/**
2093	* Sets up the VM for execution under VT-x.
2094	* This function is only called once per-VM during initialization.
2095	*
2096	* @returns VBox status code.
2097	* @param pVM Pointer to the VM.
2098	*/
2099	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2100	{
2101	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2102	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2103
2104	LogFlowFunc(("pVM=%p\n", pVM));
2105
2106	/*
2107	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2108	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2109	*/
2110	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2111	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2112	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2113	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2114	{
2115	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2116	return VERR_INTERNAL_ERROR;
2117	}
2118
2119	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2120	/*
2121	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2122	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2123	*/
2124	if ( pVM->hm.s.fAllow64BitGuests
2125	&& !HMVMX_IS_64BIT_HOST_MODE())
2126	{
2127	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2128	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2129	}
2130	#endif
2131
2132	/* Initialize these always, see hmR3InitFinalizeR0().*/
2133	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2134	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2135
2136	/* Setup the tagged-TLB flush handlers. */
2137	int rc = hmR0VmxSetupTaggedTlb(pVM);
2138	if (RT_FAILURE(rc))
2139	{
2140	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2141	return rc;
2142	}
2143
2144	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2145	{
2146	PVMCPU pVCpu = &pVM->aCpus[i];
2147	AssertPtr(pVCpu);
2148	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2149
2150	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2151	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2152
2153	/* Set revision dword at the beginning of the VMCS structure. */
2154	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2155
2156	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2157	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2158	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2159	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2160
2161	/* Load this VMCS as the current VMCS. */
2162	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2163	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2164	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2165
2166	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2167	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2168	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2169
2170	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2171	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2172	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2173
2174	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2175	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2176	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2177
2178	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2179	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2180	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2181
2182	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2183	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2184	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2185
2186	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2187	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2188	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2189	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2190	#endif
2191
2192	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2193	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2194	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2195	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2196
2197	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2198
2199	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2200	}
2201
2202	return VINF_SUCCESS;
2203	}
2204
2205
2206	/**
2207	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2208	* the VMCS.
2209	*
2210	* @returns VBox status code.
2211	* @param pVM Pointer to the VM.
2212	* @param pVCpu Pointer to the VMCPU.
2213	*/
2214	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2215	{
2216	RTCCUINTREG uReg = ASMGetCR0();
2217	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2218	AssertRCReturn(rc, rc);
2219
2220	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2221	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2222	if (HMVMX_IS_64BIT_HOST_MODE())
2223	{
2224	uint64_t uRegCR3 = HMR0Get64bitCR3();
2225	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2226	}
2227	else
2228	#endif
2229	{
2230	uReg = ASMGetCR3();
2231	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2232	}
2233	AssertRCReturn(rc, rc);
2234
2235	uReg = ASMGetCR4();
2236	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2237	AssertRCReturn(rc, rc);
2238	return rc;
2239	}
2240
2241
2242	/**
2243	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2244	* the host-state area in the VMCS.
2245	*
2246	* @returns VBox status code.
2247	* @param pVM Pointer to the VM.
2248	* @param pVCpu Pointer to the VMCPU.
2249	*/
2250	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2251	{
2252	int rc = VERR_INTERNAL_ERROR_5;
2253
2254	/*
2255	* Host DS, ES, FS and GS segment registers.
2256	*/
2257	#if HC_ARCH_BITS == 64
2258	RTSEL uSelDS = ASMGetDS();
2259	RTSEL uSelES = ASMGetES();
2260	RTSEL uSelFS = ASMGetFS();
2261	RTSEL uSelGS = ASMGetGS();
2262	#else
2263	RTSEL uSelDS = 0;
2264	RTSEL uSelES = 0;
2265	RTSEL uSelFS = 0;
2266	RTSEL uSelGS = 0;
2267	#endif
2268
2269	/* Recalculate which host-state bits need to be manually restored. */
2270	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2271
2272	/*
2273	* Host CS and SS segment registers.
2274	*/
2275	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2276	RTSEL uSelCS;
2277	RTSEL uSelSS;
2278	if (HMVMX_IS_64BIT_HOST_MODE())
2279	{
2280	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2281	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2282	}
2283	else
2284	{
2285	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2286	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2287	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2288	}
2289	#else
2290	RTSEL uSelCS = ASMGetCS();
2291	RTSEL uSelSS = ASMGetSS();
2292	#endif
2293
2294	/*
2295	* Host TR segment register.
2296	*/
2297	RTSEL uSelTR = ASMGetTR();
2298
2299	#if HC_ARCH_BITS == 64
2300	/*
2301	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2302	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2303	*/
2304	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2305	{
2306	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2307	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2308	uSelDS = 0;
2309	}
2310	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2311	{
2312	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2313	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2314	uSelES = 0;
2315	}
2316	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2317	{
2318	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2319	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2320	uSelFS = 0;
2321	}
2322	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2323	{
2324	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2325	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2326	uSelGS = 0;
2327	}
2328	#endif
2329
2330	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2331	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2332	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2333	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2334	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2335	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2336	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2337	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2338	Assert(uSelCS);
2339	Assert(uSelTR);
2340
2341	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2342	#if 0
2343	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2344	Assert(uSelSS != 0);
2345	#endif
2346
2347	/* Write these host selector fields into the host-state area in the VMCS. */
2348	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2349	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2350	#if HC_ARCH_BITS == 64
2351	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2352	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2353	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2354	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2355	#endif
2356	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2357
2358	/*
2359	* Host GDTR and IDTR.
2360	*/
2361	RTGDTR Gdtr;
2362	RT_ZERO(Gdtr);
2363	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2364	if (HMVMX_IS_64BIT_HOST_MODE())
2365	{
2366	X86XDTR64 Gdtr64;
2367	X86XDTR64 Idtr64;
2368	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2369	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2370	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2371
2372	Gdtr.cbGdt = Gdtr64.cb;
2373	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2374	}
2375	else
2376	#endif
2377	{
2378	RTIDTR Idtr;
2379	ASMGetGDTR(&Gdtr);
2380	ASMGetIDTR(&Idtr);
2381	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2382	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2383
2384	#if HC_ARCH_BITS == 64
2385	/*
2386	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2387	* maximum limit (0xffff) on every VM-exit.
2388	*/
2389	if (Gdtr.cbGdt != 0xffff)
2390	{
2391	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2392	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2393	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2394	}
2395
2396	/*
2397	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2398	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2399	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2400	*/
2401	if (Idtr.cbIdt < 0x0fff)
2402	{
2403	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2404	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2405	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2406	}
2407	#endif
2408	}
2409
2410	/*
2411	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2412	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2413	*/
2414	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2415	{
2416	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2417	return VERR_VMX_INVALID_HOST_STATE;
2418	}
2419
2420	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2421	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2422	if (HMVMX_IS_64BIT_HOST_MODE())
2423	{
2424	/* We need the 64-bit TR base for hybrid darwin. */
2425	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2426	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2427	}
2428	else
2429	#endif
2430	{
2431	uintptr_t uTRBase;
2432	#if HC_ARCH_BITS == 64
2433	uTRBase = X86DESC64_BASE(pDesc);
2434
2435	/*
2436	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2437	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2438	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2439	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2440	*
2441	* [1] See Intel spec. 3.5 "System Descriptor Types".
2442	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2443	*/
2444	Assert(pDesc->System.u4Type == 11);
2445	if ( pDesc->System.u16LimitLow != 0x67
2446	\|\| pDesc->System.u4LimitHigh)
2447	{
2448	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2449	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2450
2451	/* Store the GDTR here as we need it while restoring TR. */
2452	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2453	}
2454	#else
2455	uTRBase = X86DESC_BASE(pDesc);
2456	#endif
2457	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2458	}
2459	AssertRCReturn(rc, rc);
2460
2461	/*
2462	* Host FS base and GS base.
2463	*/
2464	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2465	if (HMVMX_IS_64BIT_HOST_MODE())
2466	{
2467	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2468	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2469	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2470	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2471
2472	# if HC_ARCH_BITS == 64
2473	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2474	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2475	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2476	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2477	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2478	# endif
2479	}
2480	#endif
2481	return rc;
2482	}
2483
2484
2485	/**
2486	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2487	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2488	* the host after every successful VM exit.
2489	*
2490	* @returns VBox status code.
2491	* @param pVM Pointer to the VM.
2492	* @param pVCpu Pointer to the VMCPU.
2493	*/
2494	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2495	{
2496	AssertPtr(pVCpu);
2497	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2498
2499	int rc = VINF_SUCCESS;
2500	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2501	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2502	uint32_t cHostMsrs = 0;
2503	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2504
2505	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2506	{
2507	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2508
2509	# if HC_ARCH_BITS == 64
2510	/* Paranoia. 64-bit code requires these bits to be set always. */
2511	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2512
2513	/*
2514	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2515	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2516	* some reason (e.g. allow transparent reads) we would activate the code below.
2517	*/
2518	# if 0
2519	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2520	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2521	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2522	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2523	if (CPUMIsGuestInLongMode(pVCpu))
2524	{
2525	uint64_t u64GuestEfer;
2526	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2527	AssertRC(rc);
2528
2529	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2530	{
2531	pHostMsr->u32Msr = MSR_K6_EFER;
2532	pHostMsr->u32Reserved = 0;
2533	pHostMsr->u64Value = u64HostEfer;
2534	pHostMsr++; cHostMsrs++;
2535	}
2536	}
2537	# endif
2538	# else /* HC_ARCH_BITS != 64 */
2539	pHostMsr->u32Msr = MSR_K6_EFER;
2540	pHostMsr->u32Reserved = 0;
2541	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2542	if (CPUMIsGuestInLongMode(pVCpu))
2543	{
2544	/* Must match the EFER value in our 64 bits switcher. */
2545	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2546	}
2547	else
2548	# endif
2549	pHostMsr->u64Value = u64HostEfer;
2550	pHostMsr++; cHostMsrs++;
2551	# endif /* HC_ARCH_BITS == 64 */
2552	}
2553
2554	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2555	if (HMVMX_IS_64BIT_HOST_MODE())
2556	{
2557	pHostMsr->u32Msr = MSR_K6_STAR;
2558	pHostMsr->u32Reserved = 0;
2559	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2560	pHostMsr++; cHostMsrs++;
2561	pHostMsr->u32Msr = MSR_K8_LSTAR;
2562	pHostMsr->u32Reserved = 0;
2563	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2564	pHostMsr++; cHostMsrs++;
2565	pHostMsr->u32Msr = MSR_K8_SF_MASK;
2566	pHostMsr->u32Reserved = 0;
2567	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2568	pHostMsr++; cHostMsrs++;
2569	pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2570	pHostMsr->u32Reserved = 0;
2571	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2572	pHostMsr++; cHostMsrs++;
2573	}
2574	# endif
2575
2576	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2577	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2578	{
2579	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2580	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2581	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2582	}
2583
2584	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2585	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2586
2587	/*
2588	* Host Sysenter MSRs.
2589	*/
2590	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2591	AssertRCReturn(rc, rc);
2592	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2593	if (HMVMX_IS_64BIT_HOST_MODE())
2594	{
2595	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2596	AssertRCReturn(rc, rc);
2597	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2598	}
2599	else
2600	{
2601	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2602	AssertRCReturn(rc, rc);
2603	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2604	}
2605	#elif HC_ARCH_BITS == 32
2606	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2607	AssertRCReturn(rc, rc);
2608	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2609	#else
2610	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2611	AssertRCReturn(rc, rc);
2612	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2613	#endif
2614	AssertRCReturn(rc, rc);
2615
2616	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2617	* hmR0VmxSetupExitCtls() !! */
2618	return rc;
2619	}
2620
2621
2622	/**
2623	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2624	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2625	* controls".
2626	*
2627	* @returns VBox status code.
2628	* @param pVCpu Pointer to the VMCPU.
2629	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2630	* out-of-sync. Make sure to update the required fields
2631	* before using them.
2632	*
2633	* @remarks No-long-jump zone!!!
2634	*/
2635	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2636	{
2637	int rc = VINF_SUCCESS;
2638	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2639	{
2640	PVM pVM = pVCpu->CTX_SUFF(pVM);
2641	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2642	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2643
2644	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2645	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2646
2647	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2648	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2649	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2650	else
2651	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2652
2653	/*
2654	* The following should -not- be set (since we're not in SMM mode):
2655	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2656	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2657	*/
2658
2659	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2660	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2661	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2662
2663	if ((val & zap) != val)
2664	{
2665	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2666	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2667	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2668	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2669	}
2670
2671	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2672	AssertRCReturn(rc, rc);
2673
2674	/* Update VCPU with the currently set VM-exit controls. */
2675	pVCpu->hm.s.vmx.u32EntryCtls = val;
2676	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2677	}
2678	return rc;
2679	}
2680
2681
2682	/**
2683	* Sets up the VM-exit controls in the VMCS.
2684	*
2685	* @returns VBox status code.
2686	* @param pVM Pointer to the VM.
2687	* @param pVCpu Pointer to the VMCPU.
2688	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2689	* out-of-sync. Make sure to update the required fields
2690	* before using them.
2691	*
2692	* @remarks requires EFER.
2693	*/
2694	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2695	{
2696	int rc = VINF_SUCCESS;
2697	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2698	{
2699	PVM pVM = pVCpu->CTX_SUFF(pVM);
2700	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2701	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2702
2703	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2704	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2705
2706	/*
2707	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2708	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2709	*/
2710	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2711	if (HMVMX_IS_64BIT_HOST_MODE())
2712	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2713	else
2714	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2715	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2716	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2717	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2718	else
2719	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2720	#endif
2721
2722	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2723	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2724
2725	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2726	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2727	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2728	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2729	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2730
2731	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2732	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2733
2734	if ((val & zap) != val)
2735	{
2736	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2737	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2738	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2739	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2740	}
2741
2742	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2743	AssertRCReturn(rc, rc);
2744
2745	/* Update VCPU with the currently set VM-exit controls. */
2746	pVCpu->hm.s.vmx.u32ExitCtls = val;
2747	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2748	}
2749	return rc;
2750	}
2751
2752
2753	/**
2754	* Loads the guest APIC and related state.
2755	*
2756	* @returns VBox status code.
2757	* @param pVM Pointer to the VM.
2758	* @param pVCpu Pointer to the VMCPU.
2759	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2760	* out-of-sync. Make sure to update the required fields
2761	* before using them.
2762	*/
2763	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2764	{
2765	int rc = VINF_SUCCESS;
2766	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2767	{
2768	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2769	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2770	{
2771	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2772
2773	bool fPendingIntr = false;
2774	uint8_t u8Tpr = 0;
2775	uint8_t u8PendingIntr = 0;
2776	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2777	AssertRCReturn(rc, rc);
2778
2779	/*
2780	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2781	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2782	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2783	* the interrupt when we VM-exit for other reasons.
2784	*/
2785	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2786	uint32_t u32TprThreshold = 0;
2787	if (fPendingIntr)
2788	{
2789	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2790	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2791	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2792	if (u8PendingPriority <= u8TprPriority)
2793	u32TprThreshold = u8PendingPriority;
2794	else
2795	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2796	}
2797	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2798
2799	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2800	AssertRCReturn(rc, rc);
2801	}
2802
2803	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
2804	}
2805	return rc;
2806	}
2807
2808
2809	/**
2810	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2811	*
2812	* @returns Guest's interruptibility-state.
2813	* @param pVCpu Pointer to the VMCPU.
2814	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2815	* out-of-sync. Make sure to update the required fields
2816	* before using them.
2817	*
2818	* @remarks No-long-jump zone!!!
2819	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2820	*/
2821	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2822	{
2823	/*
2824	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2825	* inhibit interrupts or clear any existing interrupt-inhibition.
2826	*/
2827	uint32_t uIntrState = 0;
2828	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2829	{
2830	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2831	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2832	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2833	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2834	{
2835	/*
2836	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2837	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2838	*/
2839	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2840	}
2841	else if (pMixedCtx->eflags.Bits.u1IF)
2842	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2843	else
2844	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2845	}
2846	return uIntrState;
2847	}
2848
2849
2850	/**
2851	* Loads the guest's interruptibility-state into the guest-state area in the
2852	* VMCS.
2853	*
2854	* @returns VBox status code.
2855	* @param pVCpu Pointer to the VMCPU.
2856	* @param uIntrState The interruptibility-state to set.
2857	*/
2858	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2859	{
2860	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2861	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2862	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2863	AssertRCReturn(rc, rc);
2864	return rc;
2865	}
2866
2867
2868	/**
2869	* Loads the guest's RIP into the guest-state area in the VMCS.
2870	*
2871	* @returns VBox status code.
2872	* @param pVCpu Pointer to the VMCPU.
2873	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2874	* out-of-sync. Make sure to update the required fields
2875	* before using them.
2876	*
2877	* @remarks No-long-jump zone!!!
2878	*/
2879	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2880	{
2881	int rc = VINF_SUCCESS;
2882	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
2883	{
2884	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2885	AssertRCReturn(rc, rc);
2886
2887	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
2888	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
2889	}
2890	return rc;
2891	}
2892
2893
2894	/**
2895	* Loads the guest's RSP into the guest-state area in the VMCS.
2896	*
2897	* @returns VBox status code.
2898	* @param pVCpu Pointer to the VMCPU.
2899	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2900	* out-of-sync. Make sure to update the required fields
2901	* before using them.
2902	*
2903	* @remarks No-long-jump zone!!!
2904	*/
2905	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2906	{
2907	int rc = VINF_SUCCESS;
2908	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
2909	{
2910	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2911	AssertRCReturn(rc, rc);
2912
2913	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
2914	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2915	}
2916	return rc;
2917	}
2918
2919
2920	/**
2921	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2922	*
2923	* @returns VBox status code.
2924	* @param pVCpu Pointer to the VMCPU.
2925	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2926	* out-of-sync. Make sure to update the required fields
2927	* before using them.
2928	*
2929	* @remarks No-long-jump zone!!!
2930	*/
2931	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2932	{
2933	int rc = VINF_SUCCESS;
2934	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
2935	{
2936	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2937	Let us assert it as such and use 32-bit VMWRITE. */
2938	Assert(!(pMixedCtx->rflags.u64 >> 32));
2939	X86EFLAGS Eflags = pMixedCtx->eflags;
2940	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2941	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2942
2943	/*
2944	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2945	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2946	*/
2947	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2948	{
2949	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2950	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2951	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2952	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2953	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2954	}
2955
2956	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2957	AssertRCReturn(rc, rc);
2958
2959	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
2960	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2961	}
2962	return rc;
2963	}
2964
2965
2966	/**
2967	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2968	*
2969	* @returns VBox status code.
2970	* @param pVCpu Pointer to the VMCPU.
2971	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2972	* out-of-sync. Make sure to update the required fields
2973	* before using them.
2974	*
2975	* @remarks No-long-jump zone!!!
2976	*/
2977	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2978	{
2979	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2980	AssertRCReturn(rc, rc);
2981	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2982	AssertRCReturn(rc, rc);
2983	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2984	AssertRCReturn(rc, rc);
2985	return rc;
2986	}
2987
2988
2989	/**
2990	* Loads the guest CR0 control register into the guest-state area in the VMCS.
2991	* CR0 is partially shared with the host and we have to consider the FPU bits.
2992	*
2993	* @returns VBox status code.
2994	* @param pVM Pointer to the VM.
2995	* @param pVCpu Pointer to the VMCPU.
2996	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2997	* out-of-sync. Make sure to update the required fields
2998	* before using them.
2999	*
3000	* @remarks No-long-jump zone!!!
3001	*/
3002	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3003	{
3004	/*
3005	* Guest CR0.
3006	* Guest FPU.
3007	*/
3008	int rc = VINF_SUCCESS;
3009	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3010	{
3011	Assert(!(pMixedCtx->cr0 >> 32));
3012	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3013	PVM pVM = pVCpu->CTX_SUFF(pVM);
3014
3015	/* The guest's view (read access) of its CR0 is unblemished. */
3016	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3017	AssertRCReturn(rc, rc);
3018	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3019
3020	/* Setup VT-x's view of the guest CR0. */
3021	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3022	if (pVM->hm.s.fNestedPaging)
3023	{
3024	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3025	{
3026	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3027	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3028	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3029	}
3030	else
3031	{
3032	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3033	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3034	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3035	}
3036
3037	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3038	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3039	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3040
3041	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3042	AssertRCReturn(rc, rc);
3043	}
3044	else
3045	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3046
3047	/*
3048	* Guest FPU bits.
3049	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3050	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3051	*/
3052	u32GuestCR0 \|= X86_CR0_NE;
3053	bool fInterceptNM = false;
3054	if (CPUMIsGuestFPUStateActive(pVCpu))
3055	{
3056	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3057	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3058	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3059	}
3060	else
3061	{
3062	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3063	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3064	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3065	}
3066
3067	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3068	bool fInterceptMF = false;
3069	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3070	fInterceptMF = true;
3071
3072	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3073	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3074	{
3075	Assert(PDMVmmDevHeapIsEnabled(pVM));
3076	Assert(pVM->hm.s.vmx.pRealModeTSS);
3077	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3078	fInterceptNM = true;
3079	fInterceptMF = true;
3080	}
3081	else
3082	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3083
3084	if (fInterceptNM)
3085	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3086	else
3087	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3088
3089	if (fInterceptMF)
3090	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3091	else
3092	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3093
3094	/* Additional intercepts for debugging, define these yourself explicitly. */
3095	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3096	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3097	\| RT_BIT(X86_XCPT_BP)
3098	\| RT_BIT(X86_XCPT_DB)
3099	\| RT_BIT(X86_XCPT_DE)
3100	\| RT_BIT(X86_XCPT_NM)
3101	\| RT_BIT(X86_XCPT_UD)
3102	\| RT_BIT(X86_XCPT_NP)
3103	\| RT_BIT(X86_XCPT_SS)
3104	\| RT_BIT(X86_XCPT_GP)
3105	\| RT_BIT(X86_XCPT_PF)
3106	\| RT_BIT(X86_XCPT_MF)
3107	;
3108	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3109	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3110	#endif
3111
3112	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3113
3114	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3115	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3116	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3117	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3118	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3119	else
3120	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3121
3122	u32GuestCR0 \|= uSetCR0;
3123	u32GuestCR0 &= uZapCR0;
3124	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3125
3126	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3127	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3128	AssertRCReturn(rc, rc);
3129	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3130	AssertRCReturn(rc, rc);
3131	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3132
3133	/*
3134	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3135	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3136	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3137	*/
3138	uint32_t u32CR0Mask = 0;
3139	u32CR0Mask = X86_CR0_PE
3140	\| X86_CR0_NE
3141	\| X86_CR0_WP
3142	\| X86_CR0_PG
3143	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3144	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3145	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3146
3147	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3148	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3149	* and @bugref{6944}. */
3150	#if 0
3151	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3152	u32CR0Mask &= ~X86_CR0_PE;
3153	#endif
3154	if (pVM->hm.s.fNestedPaging)
3155	u32CR0Mask &= ~X86_CR0_WP;
3156
3157	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3158	if (fInterceptNM)
3159	{
3160	u32CR0Mask \|= X86_CR0_TS
3161	\| X86_CR0_MP;
3162	}
3163
3164	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3165	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3166	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3167	AssertRCReturn(rc, rc);
3168	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3169
3170	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3171	}
3172	return rc;
3173	}
3174
3175
3176	/**
3177	* Loads the guest control registers (CR3, CR4) into the guest-state area
3178	* in the VMCS.
3179	*
3180	* @returns VBox status code.
3181	* @param pVM Pointer to the VM.
3182	* @param pVCpu Pointer to the VMCPU.
3183	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3184	* out-of-sync. Make sure to update the required fields
3185	* before using them.
3186	*
3187	* @remarks No-long-jump zone!!!
3188	*/
3189	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3190	{
3191	int rc = VINF_SUCCESS;
3192	PVM pVM = pVCpu->CTX_SUFF(pVM);
3193
3194	/*
3195	* Guest CR2.
3196	* It's always loaded in the assembler code. Nothing to do here.
3197	*/
3198
3199	/*
3200	* Guest CR3.
3201	*/
3202	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3203	{
3204	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3205	if (pVM->hm.s.fNestedPaging)
3206	{
3207	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3208
3209	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3210	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3211	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3212	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3213
3214	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3215	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3216	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3217
3218	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3219	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3220	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3221	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3222
3223	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3224	AssertRCReturn(rc, rc);
3225	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3226
3227	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3228	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3229	{
3230	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3231	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3232	{
3233	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3234	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3235	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3236	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3237	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3238	}
3239
3240	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3241	have Unrestricted Execution to handle the guest when it's not using paging. */
3242	GCPhysGuestCR3 = pMixedCtx->cr3;
3243	}
3244	else
3245	{
3246	/*
3247	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3248	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3249	* EPT takes care of translating it to host-physical addresses.
3250	*/
3251	RTGCPHYS GCPhys;
3252	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3253	Assert(PDMVmmDevHeapIsEnabled(pVM));
3254
3255	/* We obtain it here every time as the guest could have relocated this PCI region. */
3256	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3257	AssertRCReturn(rc, rc);
3258
3259	GCPhysGuestCR3 = GCPhys;
3260	}
3261
3262	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3263	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3264	}
3265	else
3266	{
3267	/* Non-nested paging case, just use the hypervisor's CR3. */
3268	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3269
3270	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3271	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3272	}
3273	AssertRCReturn(rc, rc);
3274
3275	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3276	}
3277
3278	/*
3279	* Guest CR4.
3280	*/
3281	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3282	{
3283	Assert(!(pMixedCtx->cr4 >> 32));
3284	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3285
3286	/* The guest's view of its CR4 is unblemished. */
3287	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3288	AssertRCReturn(rc, rc);
3289	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3290
3291	/* Setup VT-x's view of the guest CR4. */
3292	/*
3293	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3294	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3295	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3296	*/
3297	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3298	{
3299	Assert(pVM->hm.s.vmx.pRealModeTSS);
3300	Assert(PDMVmmDevHeapIsEnabled(pVM));
3301	u32GuestCR4 &= ~X86_CR4_VME;
3302	}
3303
3304	if (pVM->hm.s.fNestedPaging)
3305	{
3306	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3307	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3308	{
3309	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3310	u32GuestCR4 \|= X86_CR4_PSE;
3311	/* Our identity mapping is a 32 bits page directory. */
3312	u32GuestCR4 &= ~X86_CR4_PAE;
3313	}
3314	/* else use guest CR4.*/
3315	}
3316	else
3317	{
3318	/*
3319	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3320	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3321	*/
3322	switch (pVCpu->hm.s.enmShadowMode)
3323	{
3324	case PGMMODE_REAL: /* Real-mode. */
3325	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3326	case PGMMODE_32_BIT: /* 32-bit paging. */
3327	{
3328	u32GuestCR4 &= ~X86_CR4_PAE;
3329	break;
3330	}
3331
3332	case PGMMODE_PAE: /* PAE paging. */
3333	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3334	{
3335	u32GuestCR4 \|= X86_CR4_PAE;
3336	break;
3337	}
3338
3339	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3340	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3341	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3342	break;
3343	#endif
3344	default:
3345	AssertFailed();
3346	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3347	}
3348	}
3349
3350	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3351	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3352	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3353	u32GuestCR4 \|= uSetCR4;
3354	u32GuestCR4 &= uZapCR4;
3355
3356	/* Write VT-x's view of the guest CR4 into the VMCS. */
3357	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3358	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3359	AssertRCReturn(rc, rc);
3360
3361	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3362	uint32_t u32CR4Mask = 0;
3363	u32CR4Mask = X86_CR4_VME
3364	\| X86_CR4_PAE
3365	\| X86_CR4_PGE
3366	\| X86_CR4_PSE
3367	\| X86_CR4_VMXE;
3368	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3369	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3370	AssertRCReturn(rc, rc);
3371
3372	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3373	}
3374	return rc;
3375	}
3376
3377
3378	/**
3379	* Loads the guest debug registers into the guest-state area in the VMCS.
3380	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3381	*
3382	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3383	*
3384	* @returns VBox status code.
3385	* @param pVCpu Pointer to the VMCPU.
3386	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3387	* out-of-sync. Make sure to update the required fields
3388	* before using them.
3389	*
3390	* @remarks No-long-jump zone!!!
3391	*/
3392	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3393	{
3394	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3395	return VINF_SUCCESS;
3396
3397	#ifdef VBOX_STRICT
3398	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3399	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3400	{
3401	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3402	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3403	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3404	}
3405	#endif
3406
3407	int rc;
3408	PVM pVM = pVCpu->CTX_SUFF(pVM);
3409	bool fInterceptDB = false;
3410	bool fInterceptMovDRx = false;
3411	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3412	{
3413	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3414	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3415	{
3416	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3417	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3418	AssertRCReturn(rc, rc);
3419	Assert(fInterceptDB == false);
3420	}
3421	else
3422	{
3423	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3424	pVCpu->hm.s.fClearTrapFlag = true;
3425	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3426	fInterceptDB = true;
3427	}
3428	}
3429
3430	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3431	{
3432	/*
3433	* Use the combined guest and host DRx values found in the hypervisor
3434	* register set because the debugger has breakpoints active or someone
3435	* is single stepping on the host side without a monitor trap flag.
3436	*
3437	* Note! DBGF expects a clean DR6 state before executing guest code.
3438	*/
3439	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3440	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3441	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3442	{
3443	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3444	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3445	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3446	}
3447	else
3448	#endif
3449	if (!CPUMIsHyperDebugStateActive(pVCpu))
3450	{
3451	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3452	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3453	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3454	}
3455
3456	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3457	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3458	AssertRCReturn(rc, rc);
3459
3460	pVCpu->hm.s.fUsingHyperDR7 = true;
3461	fInterceptDB = true;
3462	fInterceptMovDRx = true;
3463	}
3464	else
3465	{
3466	/*
3467	* If the guest has enabled debug registers, we need to load them prior to
3468	* executing guest code so they'll trigger at the right time.
3469	*/
3470	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3471	{
3472	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3473	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3474	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3475	{
3476	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3477	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3478	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3479	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3480	}
3481	else
3482	#endif
3483	if (CPUMIsGuestDebugStateActive(pVCpu))
3484	{
3485	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3486	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3487	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3488	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3489	}
3490	}
3491	/*
3492	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3493	* must intercept #DB in order to maintain a correct DR6 guest value.
3494	*/
3495	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3496	else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3497	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3498	\|\| !CPUMIsGuestDebugStateActive(pVCpu))
3499	#else
3500	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3501	#endif
3502	{
3503	fInterceptMovDRx = true;
3504	fInterceptDB = true;
3505	}
3506
3507	/* Update guest DR7. */
3508	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3509	AssertRCReturn(rc, rc);
3510
3511	pVCpu->hm.s.fUsingHyperDR7 = false;
3512	}
3513
3514	/*
3515	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3516	*/
3517	if (fInterceptDB)
3518	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3519	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3520	{
3521	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3522	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3523	#endif
3524	}
3525	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3526	AssertRCReturn(rc, rc);
3527
3528	/*
3529	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3530	*/
3531	if (fInterceptMovDRx)
3532	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3533	else
3534	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3535	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3536	AssertRCReturn(rc, rc);
3537
3538	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3539	return VINF_SUCCESS;
3540	}
3541
3542
3543	#ifdef VBOX_STRICT
3544	/**
3545	* Strict function to validate segment registers.
3546	*
3547	* @remarks ASSUMES CR0 is up to date.
3548	*/
3549	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3550	{
3551	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3552	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3553	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3554	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3555	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3556	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3557	{
3558	/* Protected mode checks */
3559	/* CS */
3560	Assert(pCtx->cs.Attr.n.u1Present);
3561	Assert(!(pCtx->cs.Attr.u & 0xf00));
3562	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3563	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3564	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3565	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3566	\|\| (pCtx->cs.Attr.n.u1Granularity));
3567	/* CS cannot be loaded with NULL in protected mode. */
3568	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3569	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3570	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3571	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3572	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3573	else
3574	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3575	/* SS */
3576	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3577	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3578	if ( !(pCtx->cr0 & X86_CR0_PE)
3579	\|\| pCtx->cs.Attr.n.u4Type == 3)
3580	{
3581	Assert(!pCtx->ss.Attr.n.u2Dpl);
3582	}
3583	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3584	{
3585	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3586	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3587	Assert(pCtx->ss.Attr.n.u1Present);
3588	Assert(!(pCtx->ss.Attr.u & 0xf00));
3589	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3590	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3591	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3592	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3593	\|\| (pCtx->ss.Attr.n.u1Granularity));
3594	}
3595	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3596	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3597	{
3598	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3599	Assert(pCtx->ds.Attr.n.u1Present);
3600	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3601	Assert(!(pCtx->ds.Attr.u & 0xf00));
3602	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3603	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3604	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3605	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3606	\|\| (pCtx->ds.Attr.n.u1Granularity));
3607	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3608	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3609	}
3610	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3611	{
3612	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3613	Assert(pCtx->es.Attr.n.u1Present);
3614	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3615	Assert(!(pCtx->es.Attr.u & 0xf00));
3616	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3617	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3618	\|\| !(pCtx->es.Attr.n.u1Granularity));
3619	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3620	\|\| (pCtx->es.Attr.n.u1Granularity));
3621	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3622	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3623	}
3624	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3625	{
3626	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3627	Assert(pCtx->fs.Attr.n.u1Present);
3628	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3629	Assert(!(pCtx->fs.Attr.u & 0xf00));
3630	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3631	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3632	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3633	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3634	\|\| (pCtx->fs.Attr.n.u1Granularity));
3635	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3636	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3637	}
3638	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3639	{
3640	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3641	Assert(pCtx->gs.Attr.n.u1Present);
3642	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3643	Assert(!(pCtx->gs.Attr.u & 0xf00));
3644	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3645	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3646	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3647	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3648	\|\| (pCtx->gs.Attr.n.u1Granularity));
3649	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3650	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3651	}
3652	/* 64-bit capable CPUs. */
3653	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3654	Assert(!(pCtx->cs.u64Base >> 32));
3655	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3656	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3657	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3658	# endif
3659	}
3660	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3661	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3662	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3663	{
3664	/* Real and v86 mode checks. */
3665	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3666	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3667	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3668	{
3669	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3670	}
3671	else
3672	{
3673	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3674	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3675	}
3676
3677	/* CS */
3678	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3679	Assert(pCtx->cs.u32Limit == 0xffff);
3680	Assert(u32CSAttr == 0xf3);
3681	/* SS */
3682	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3683	Assert(pCtx->ss.u32Limit == 0xffff);
3684	Assert(u32SSAttr == 0xf3);
3685	/* DS */
3686	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3687	Assert(pCtx->ds.u32Limit == 0xffff);
3688	Assert(u32DSAttr == 0xf3);
3689	/* ES */
3690	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3691	Assert(pCtx->es.u32Limit == 0xffff);
3692	Assert(u32ESAttr == 0xf3);
3693	/* FS */
3694	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3695	Assert(pCtx->fs.u32Limit == 0xffff);
3696	Assert(u32FSAttr == 0xf3);
3697	/* GS */
3698	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3699	Assert(pCtx->gs.u32Limit == 0xffff);
3700	Assert(u32GSAttr == 0xf3);
3701	/* 64-bit capable CPUs. */
3702	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3703	Assert(!(pCtx->cs.u64Base >> 32));
3704	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3705	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3706	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3707	# endif
3708	}
3709	}
3710	#endif /* VBOX_STRICT */
3711
3712
3713	/**
3714	* Writes a guest segment register into the guest-state area in the VMCS.
3715	*
3716	* @returns VBox status code.
3717	* @param pVCpu Pointer to the VMCPU.
3718	* @param idxSel Index of the selector in the VMCS.
3719	* @param idxLimit Index of the segment limit in the VMCS.
3720	* @param idxBase Index of the segment base in the VMCS.
3721	* @param idxAccess Index of the access rights of the segment in the VMCS.
3722	* @param pSelReg Pointer to the segment selector.
3723	* @param pCtx Pointer to the guest-CPU context.
3724	*
3725	* @remarks No-long-jump zone!!!
3726	*/
3727	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3728	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3729	{
3730	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3731	AssertRCReturn(rc, rc);
3732	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3733	AssertRCReturn(rc, rc);
3734	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3735	AssertRCReturn(rc, rc);
3736
3737	uint32_t u32Access = pSelReg->Attr.u;
3738	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3739	{
3740	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3741	u32Access = 0xf3;
3742	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3743	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3744	}
3745	else
3746	{
3747	/*
3748	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3749	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3750	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3751	* loaded in protected-mode have their attribute as 0.
3752	*/
3753	if (!u32Access)
3754	u32Access = X86DESCATTR_UNUSABLE;
3755	}
3756
3757	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3758	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3759	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3760
3761	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3762	AssertRCReturn(rc, rc);
3763	return rc;
3764	}
3765
3766
3767	/**
3768	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3769	* into the guest-state area in the VMCS.
3770	*
3771	* @returns VBox status code.
3772	* @param pVM Pointer to the VM.
3773	* @param pVCPU Pointer to the VMCPU.
3774	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3775	* out-of-sync. Make sure to update the required fields
3776	* before using them.
3777	*
3778	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3779	* @remarks No-long-jump zone!!!
3780	*/
3781	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3782	{
3783	int rc = VERR_INTERNAL_ERROR_5;
3784	PVM pVM = pVCpu->CTX_SUFF(pVM);
3785
3786	/*
3787	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3788	*/
3789	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
3790	{
3791	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3792	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3793	{
3794	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3795	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3796	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3797	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3798	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3799	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3800	}
3801
3802	#ifdef VBOX_WITH_REM
3803	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3804	{
3805	Assert(pVM->hm.s.vmx.pRealModeTSS);
3806	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3807	if ( pVCpu->hm.s.vmx.fWasInRealMode
3808	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3809	{
3810	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3811	in real-mode (e.g. OpenBSD 4.0) */
3812	REMFlushTBs(pVM);
3813	Log4(("Load: Switch to protected mode detected!\n"));
3814	pVCpu->hm.s.vmx.fWasInRealMode = false;
3815	}
3816	}
3817	#endif
3818	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3819	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3820	AssertRCReturn(rc, rc);
3821	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3822	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3823	AssertRCReturn(rc, rc);
3824	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3825	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3826	AssertRCReturn(rc, rc);
3827	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3828	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3829	AssertRCReturn(rc, rc);
3830	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3831	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3832	AssertRCReturn(rc, rc);
3833	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3834	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3835	AssertRCReturn(rc, rc);
3836
3837	#ifdef VBOX_STRICT
3838	/* Validate. */
3839	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3840	#endif
3841
3842	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
3843	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3844	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3845	}
3846
3847	/*
3848	* Guest TR.
3849	*/
3850	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
3851	{
3852	/*
3853	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3854	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3855	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3856	*/
3857	uint16_t u16Sel = 0;
3858	uint32_t u32Limit = 0;
3859	uint64_t u64Base = 0;
3860	uint32_t u32AccessRights = 0;
3861
3862	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3863	{
3864	u16Sel = pMixedCtx->tr.Sel;
3865	u32Limit = pMixedCtx->tr.u32Limit;
3866	u64Base = pMixedCtx->tr.u64Base;
3867	u32AccessRights = pMixedCtx->tr.Attr.u;
3868	}
3869	else
3870	{
3871	Assert(pVM->hm.s.vmx.pRealModeTSS);
3872	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3873
3874	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3875	RTGCPHYS GCPhys;
3876	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3877	AssertRCReturn(rc, rc);
3878
3879	X86DESCATTR DescAttr;
3880	DescAttr.u = 0;
3881	DescAttr.n.u1Present = 1;
3882	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3883
3884	u16Sel = 0;
3885	u32Limit = HM_VTX_TSS_SIZE;
3886	u64Base = GCPhys; /* in real-mode phys = virt. */
3887	u32AccessRights = DescAttr.u;
3888	}
3889
3890	/* Validate. */
3891	Assert(!(u16Sel & RT_BIT(2)));
3892	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3893	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3894	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3895	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3896	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3897	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3898	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3899	Assert( (u32Limit & 0xfff) == 0xfff
3900	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3901	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3902	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3903
3904	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3905	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3906	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3907	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3908
3909	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
3910	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3911	}
3912
3913	/*
3914	* Guest GDTR.
3915	*/
3916	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
3917	{
3918	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3919	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3920
3921	/* Validate. */
3922	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3923
3924	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
3925	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3926	}
3927
3928	/*
3929	* Guest LDTR.
3930	*/
3931	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
3932	{
3933	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3934	uint32_t u32Access = 0;
3935	if (!pMixedCtx->ldtr.Attr.u)
3936	u32Access = X86DESCATTR_UNUSABLE;
3937	else
3938	u32Access = pMixedCtx->ldtr.Attr.u;
3939
3940	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3941	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3942	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3943	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3944
3945	/* Validate. */
3946	if (!(u32Access & X86DESCATTR_UNUSABLE))
3947	{
3948	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3949	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3950	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3951	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3952	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3953	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3954	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3955	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3956	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3957	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3958	}
3959
3960	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
3961	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3962	}
3963
3964	/*
3965	* Guest IDTR.
3966	*/
3967	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
3968	{
3969	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3970	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3971
3972	/* Validate. */
3973	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3974
3975	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
3976	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3977	}
3978
3979	return VINF_SUCCESS;
3980	}
3981
3982
3983	/**
3984	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3985	* areas. These MSRs will automatically be loaded to the host CPU on every
3986	* successful VM entry and stored from the host CPU on every successful VM exit.
3987	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3988	*
3989	* @returns VBox status code.
3990	* @param pVCpu Pointer to the VMCPU.
3991	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3992	* out-of-sync. Make sure to update the required fields
3993	* before using them.
3994	*
3995	* @remarks No-long-jump zone!!!
3996	*/
3997	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3998	{
3999	AssertPtr(pVCpu);
4000	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4001
4002	/*
4003	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4004	*/
4005	int rc = VINF_SUCCESS;
4006	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4007	{
4008	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4009	PVM pVM = pVCpu->CTX_SUFF(pVM);
4010	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4011	uint32_t cGuestMsrs = 0;
4012
4013	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4014	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4015	* when the guest really is in 64-bit mode. */
4016	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4017	if (fSupportsLongMode)
4018	{
4019	pGuestMsr->u32Msr = MSR_K8_LSTAR;
4020	pGuestMsr->u32Reserved = 0;
4021	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4022	pGuestMsr++; cGuestMsrs++;
4023	pGuestMsr->u32Msr = MSR_K6_STAR;
4024	pGuestMsr->u32Reserved = 0;
4025	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4026	pGuestMsr++; cGuestMsrs++;
4027	pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4028	pGuestMsr->u32Reserved = 0;
4029	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4030	pGuestMsr++; cGuestMsrs++;
4031	pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4032	pGuestMsr->u32Reserved = 0;
4033	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4034	pGuestMsr++; cGuestMsrs++;
4035	}
4036
4037	/*
4038	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4039	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
4040	*/
4041	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
4042	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
4043	{
4044	pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4045	pGuestMsr->u32Reserved = 0;
4046	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4047	AssertRCReturn(rc, rc);
4048	pGuestMsr++; cGuestMsrs++;
4049	}
4050
4051	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4052	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4053	{
4054	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4055	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4056	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4057	}
4058
4059	/* Update the VCPU's copy of the guest MSR count. */
4060	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4061	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4062	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4063	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4064
4065	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4066	}
4067
4068	/*
4069	* Guest Sysenter MSRs.
4070	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4071	* VM-exits on WRMSRs for these MSRs.
4072	*/
4073	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4074	{
4075	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4076	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4077	}
4078
4079	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4080	{
4081	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4082	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4083	}
4084
4085	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4086	{
4087	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4088	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4089	}
4090
4091	return rc;
4092	}
4093
4094
4095	/**
4096	* Loads the guest activity state into the guest-state area in the VMCS.
4097	*
4098	* @returns VBox status code.
4099	* @param pVCpu Pointer to the VMCPU.
4100	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4101	* out-of-sync. Make sure to update the required fields
4102	* before using them.
4103	*
4104	* @remarks No-long-jump zone!!!
4105	*/
4106	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4107	{
4108	/** @todo See if we can make use of other states, e.g.
4109	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4110	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4111	{
4112	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4113	AssertRCReturn(rc, rc);
4114
4115	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4116	}
4117	return VINF_SUCCESS;
4118	}
4119
4120
4121	/**
4122	* Sets up the appropriate function to run guest code.
4123	*
4124	* @returns VBox status code.
4125	* @param pVCpu Pointer to the VMCPU.
4126	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4127	* out-of-sync. Make sure to update the required fields
4128	* before using them.
4129	*
4130	* @remarks No-long-jump zone!!!
4131	*/
4132	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4133	{
4134	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4135	{
4136	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4137	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4138	#endif
4139	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4140	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4141	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4142	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4143	{
4144	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4145	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4146	}
4147	#else
4148	/* 64-bit host or hybrid host. */
4149	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4150	#endif
4151	}
4152	else
4153	{
4154	/* Guest is not in long mode, use the 32-bit handler. */
4155	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4156	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4157	{
4158	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4159	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4160	}
4161	#else
4162	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4163	#endif
4164	}
4165	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4166	return VINF_SUCCESS;
4167	}
4168
4169
4170	/**
4171	* Wrapper for running the guest code in VT-x.
4172	*
4173	* @returns VBox strict status code.
4174	* @param pVM Pointer to the VM.
4175	* @param pVCpu Pointer to the VMCPU.
4176	* @param pCtx Pointer to the guest-CPU context.
4177	*
4178	* @remarks No-long-jump zone!!!
4179	*/
4180	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4181	{
4182	/*
4183	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4184	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4185	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4186	*/
4187	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4188	/** @todo Add stats for resume vs launch. */
4189	#ifdef VBOX_WITH_KERNEL_USING_XMM
4190	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4191	#else
4192	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4193	#endif
4194	}
4195
4196
4197	/**
4198	* Reports world-switch error and dumps some useful debug info.
4199	*
4200	* @param pVM Pointer to the VM.
4201	* @param pVCpu Pointer to the VMCPU.
4202	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4203	* @param pCtx Pointer to the guest-CPU context.
4204	* @param pVmxTransient Pointer to the VMX transient structure (only
4205	* exitReason updated).
4206	*/
4207	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4208	{
4209	Assert(pVM);
4210	Assert(pVCpu);
4211	Assert(pCtx);
4212	Assert(pVmxTransient);
4213	HMVMX_ASSERT_PREEMPT_SAFE();
4214
4215	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4216	switch (rcVMRun)
4217	{
4218	case VERR_VMX_INVALID_VMXON_PTR:
4219	AssertFailed();
4220	break;
4221	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4222	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4223	{
4224	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4225	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4226	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4227	AssertRC(rc);
4228
4229	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4230	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4231	Cannot do it here as we may have been long preempted. */
4232
4233	#ifdef VBOX_STRICT
4234	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4235	pVmxTransient->uExitReason));
4236	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4237	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4238	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4239	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4240	else
4241	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4242	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4243	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4244
4245	/* VMX control bits. */
4246	uint32_t u32Val;
4247	uint64_t u64Val;
4248	HMVMXHCUINTREG uHCReg;
4249	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4250	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4251	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4252	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4253	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4254	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4255	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4256	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4257	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4258	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4259	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4260	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4261	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4262	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4263	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4264	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4265	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4266	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4267	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4268	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4269	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4270	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4271	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4272	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4273	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4274	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4275	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4276	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4277	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4278	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4279	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4280	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4281	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4282	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4283	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4284	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4285	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4286	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4287	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4288	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4289	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4290	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4291
4292	/* Guest bits. */
4293	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4294	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4295	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4296	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4297	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4298	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4299	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4300	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4301
4302	/* Host bits. */
4303	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4304	Log4(("Host CR0 %#RHr\n", uHCReg));
4305	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4306	Log4(("Host CR3 %#RHr\n", uHCReg));
4307	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4308	Log4(("Host CR4 %#RHr\n", uHCReg));
4309
4310	RTGDTR HostGdtr;
4311	PCX86DESCHC pDesc;
4312	ASMGetGDTR(&HostGdtr);
4313	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4314	Log4(("Host CS %#08x\n", u32Val));
4315	if (u32Val < HostGdtr.cbGdt)
4316	{
4317	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4318	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4319	}
4320
4321	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4322	Log4(("Host DS %#08x\n", u32Val));
4323	if (u32Val < HostGdtr.cbGdt)
4324	{
4325	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4326	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4327	}
4328
4329	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4330	Log4(("Host ES %#08x\n", u32Val));
4331	if (u32Val < HostGdtr.cbGdt)
4332	{
4333	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4334	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4335	}
4336
4337	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4338	Log4(("Host FS %#08x\n", u32Val));
4339	if (u32Val < HostGdtr.cbGdt)
4340	{
4341	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4342	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4343	}
4344
4345	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4346	Log4(("Host GS %#08x\n", u32Val));
4347	if (u32Val < HostGdtr.cbGdt)
4348	{
4349	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4350	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4351	}
4352
4353	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4354	Log4(("Host SS %#08x\n", u32Val));
4355	if (u32Val < HostGdtr.cbGdt)
4356	{
4357	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4358	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4359	}
4360
4361	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4362	Log4(("Host TR %#08x\n", u32Val));
4363	if (u32Val < HostGdtr.cbGdt)
4364	{
4365	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4366	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4367	}
4368
4369	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4370	Log4(("Host TR Base %#RHv\n", uHCReg));
4371	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4372	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4373	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4374	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4375	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4376	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4377	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4378	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4379	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4380	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4381	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4382	Log4(("Host RSP %#RHv\n", uHCReg));
4383	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4384	Log4(("Host RIP %#RHv\n", uHCReg));
4385	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4386	if (HMVMX_IS_64BIT_HOST_MODE())
4387	{
4388	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4389	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4390	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4391	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4392	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4393	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4394	}
4395	# endif
4396	#endif /* VBOX_STRICT */
4397	break;
4398	}
4399
4400	default:
4401	/* Impossible */
4402	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4403	break;
4404	}
4405	NOREF(pVM);
4406	}
4407
4408
4409	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4410	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4411	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4412	#endif
4413	#ifdef VBOX_STRICT
4414	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4415	{
4416	switch (idxField)
4417	{
4418	case VMX_VMCS_GUEST_RIP:
4419	case VMX_VMCS_GUEST_RSP:
4420	case VMX_VMCS_GUEST_SYSENTER_EIP:
4421	case VMX_VMCS_GUEST_SYSENTER_ESP:
4422	case VMX_VMCS_GUEST_GDTR_BASE:
4423	case VMX_VMCS_GUEST_IDTR_BASE:
4424	case VMX_VMCS_GUEST_CS_BASE:
4425	case VMX_VMCS_GUEST_DS_BASE:
4426	case VMX_VMCS_GUEST_ES_BASE:
4427	case VMX_VMCS_GUEST_FS_BASE:
4428	case VMX_VMCS_GUEST_GS_BASE:
4429	case VMX_VMCS_GUEST_SS_BASE:
4430	case VMX_VMCS_GUEST_LDTR_BASE:
4431	case VMX_VMCS_GUEST_TR_BASE:
4432	case VMX_VMCS_GUEST_CR3:
4433	return true;
4434	}
4435	return false;
4436	}
4437
4438	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4439	{
4440	switch (idxField)
4441	{
4442	/* Read-only fields. */
4443	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4444	return true;
4445	}
4446	/* Remaining readable fields should also be writable. */
4447	return hmR0VmxIsValidWriteField(idxField);
4448	}
4449	#endif /* VBOX_STRICT */
4450
4451
4452	/**
4453	* Executes the specified handler in 64-bit mode.
4454	*
4455	* @returns VBox status code.
4456	* @param pVM Pointer to the VM.
4457	* @param pVCpu Pointer to the VMCPU.
4458	* @param pCtx Pointer to the guest CPU context.
4459	* @param enmOp The operation to perform.
4460	* @param cbParam Number of parameters.
4461	* @param paParam Array of 32-bit parameters.
4462	*/
4463	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4464	uint32_t *paParam)
4465	{
4466	int rc, rc2;
4467	PHMGLOBALCPUINFO pCpu;
4468	RTHCPHYS HCPhysCpuPage;
4469	RTCCUINTREG uOldEflags;
4470
4471	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4472	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4473	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4474	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4475
4476	#ifdef VBOX_STRICT
4477	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4478	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4479
4480	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4481	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4482	#endif
4483
4484	/* Disable interrupts. */
4485	uOldEflags = ASMIntDisableFlags();
4486
4487	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4488	RTCPUID idHostCpu = RTMpCpuId();
4489	CPUMR0SetLApic(pVCpu, idHostCpu);
4490	#endif
4491
4492	pCpu = HMR0GetCurrentCpu();
4493	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4494
4495	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4496	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4497
4498	/* Leave VMX Root Mode. */
4499	VMXDisable();
4500
4501	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4502
4503	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4504	CPUMSetHyperEIP(pVCpu, enmOp);
4505	for (int i = (int)cbParam - 1; i >= 0; i--)
4506	CPUMPushHyper(pVCpu, paParam[i]);
4507
4508	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4509
4510	/* Call the switcher. */
4511	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4512	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4513
4514	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4515	/* Make sure the VMX instructions don't cause #UD faults. */
4516	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4517
4518	/* Re-enter VMX Root Mode */
4519	rc2 = VMXEnable(HCPhysCpuPage);
4520	if (RT_FAILURE(rc2))
4521	{
4522	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4523	ASMSetFlags(uOldEflags);
4524	return rc2;
4525	}
4526
4527	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4528	AssertRC(rc2);
4529	Assert(!(ASMGetFlags() & X86_EFL_IF));
4530	ASMSetFlags(uOldEflags);
4531	return rc;
4532	}
4533
4534
4535	/**
4536	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4537	* supporting 64-bit guests.
4538	*
4539	* @returns VBox status code.
4540	* @param fResume Whether to VMLAUNCH or VMRESUME.
4541	* @param pCtx Pointer to the guest-CPU context.
4542	* @param pCache Pointer to the VMCS cache.
4543	* @param pVM Pointer to the VM.
4544	* @param pVCpu Pointer to the VMCPU.
4545	*/
4546	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4547	{
4548	uint32_t aParam[6];
4549	PHMGLOBALCPUINFO pCpu = NULL;
4550	RTHCPHYS HCPhysCpuPage = 0;
4551	int rc = VERR_INTERNAL_ERROR_5;
4552
4553	pCpu = HMR0GetCurrentCpu();
4554	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4555
4556	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4557	pCache->uPos = 1;
4558	pCache->interPD = PGMGetInterPaeCR3(pVM);
4559	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4560	#endif
4561
4562	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4563	pCache->TestIn.HCPhysCpuPage = 0;
4564	pCache->TestIn.HCPhysVmcs = 0;
4565	pCache->TestIn.pCache = 0;
4566	pCache->TestOut.HCPhysVmcs = 0;
4567	pCache->TestOut.pCache = 0;
4568	pCache->TestOut.pCtx = 0;
4569	pCache->TestOut.eflags = 0;
4570	#endif
4571
4572	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4573	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4574	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4575	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4576	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4577	aParam[5] = 0;
4578
4579	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4580	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4581	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4582	#endif
4583	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4584
4585	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4586	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4587	Assert(pCtx->dr[4] == 10);
4588	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4589	#endif
4590
4591	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4592	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4593	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4594	pVCpu->hm.s.vmx.HCPhysVmcs));
4595	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4596	pCache->TestOut.HCPhysVmcs));
4597	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4598	pCache->TestOut.pCache));
4599	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4600	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4601	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4602	pCache->TestOut.pCtx));
4603	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4604	#endif
4605	return rc;
4606	}
4607
4608
4609	/**
4610	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4611	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4612	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4613	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4614	*
4615	* @returns VBox status code.
4616	* @param pVM Pointer to the VM.
4617	* @param pVCpu Pointer to the VMCPU.
4618	*/
4619	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4620	{
4621	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4622	{ \
4623	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4624	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4625	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4626	++cReadFields; \
4627	}
4628
4629	AssertPtr(pVM);
4630	AssertPtr(pVCpu);
4631	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4632	uint32_t cReadFields = 0;
4633
4634	/*
4635	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4636	* and serve to indicate exceptions to the rules.
4637	*/
4638
4639	/* Guest-natural selector base fields. */
4640	#if 0
4641	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4642	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4643	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4644	#endif
4645	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4646	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4647	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4648	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4649	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4650	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4651	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4652	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4653	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4654	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4655	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4656	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4657	#if 0
4658	/* Unused natural width guest-state fields. */
4659	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4660	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4661	#endif
4662	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4663	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4664
4665	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4666	#if 0
4667	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4668	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4669	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4670	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4671	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4672	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4673	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4674	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4675	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4676	#endif
4677
4678	/* Natural width guest-state fields. */
4679	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4680	#if 0
4681	/* Currently unused field. */
4682	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4683	#endif
4684
4685	if (pVM->hm.s.fNestedPaging)
4686	{
4687	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4688	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4689	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4690	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4691	}
4692	else
4693	{
4694	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4695	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4696	}
4697
4698	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4699	return VINF_SUCCESS;
4700	}
4701
4702
4703	/**
4704	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4705	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4706	* darwin, running 64-bit guests).
4707	*
4708	* @returns VBox status code.
4709	* @param pVCpu Pointer to the VMCPU.
4710	* @param idxField The VMCS field encoding.
4711	* @param u64Val 16, 32 or 64 bits value.
4712	*/
4713	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4714	{
4715	int rc;
4716	switch (idxField)
4717	{
4718	/*
4719	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4720	*/
4721	/* 64-bit Control fields. */
4722	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4723	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4724	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4725	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4726	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4727	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4728	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4729	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4730	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4731	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4732	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4733	case VMX_VMCS64_CTRL_EPTP_FULL:
4734	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4735	/* 64-bit Guest-state fields. */
4736	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4737	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4738	case VMX_VMCS64_GUEST_PAT_FULL:
4739	case VMX_VMCS64_GUEST_EFER_FULL:
4740	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4741	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4742	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4743	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4744	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4745	/* 64-bit Host-state fields. */
4746	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4747	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4748	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4749	{
4750	rc = VMXWriteVmcs32(idxField, u64Val);
4751	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4752	break;
4753	}
4754
4755	/*
4756	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4757	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4758	*/
4759	/* Natural-width Guest-state fields. */
4760	case VMX_VMCS_GUEST_CR3:
4761	case VMX_VMCS_GUEST_ES_BASE:
4762	case VMX_VMCS_GUEST_CS_BASE:
4763	case VMX_VMCS_GUEST_SS_BASE:
4764	case VMX_VMCS_GUEST_DS_BASE:
4765	case VMX_VMCS_GUEST_FS_BASE:
4766	case VMX_VMCS_GUEST_GS_BASE:
4767	case VMX_VMCS_GUEST_LDTR_BASE:
4768	case VMX_VMCS_GUEST_TR_BASE:
4769	case VMX_VMCS_GUEST_GDTR_BASE:
4770	case VMX_VMCS_GUEST_IDTR_BASE:
4771	case VMX_VMCS_GUEST_RSP:
4772	case VMX_VMCS_GUEST_RIP:
4773	case VMX_VMCS_GUEST_SYSENTER_ESP:
4774	case VMX_VMCS_GUEST_SYSENTER_EIP:
4775	{
4776	if (!(u64Val >> 32))
4777	{
4778	/* If this field is 64-bit, VT-x will zero out the top bits. */
4779	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4780	}
4781	else
4782	{
4783	/* Assert that only the 32->64 switcher case should ever come here. */
4784	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4785	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4786	}
4787	break;
4788	}
4789
4790	default:
4791	{
4792	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4793	rc = VERR_INVALID_PARAMETER;
4794	break;
4795	}
4796	}
4797	AssertRCReturn(rc, rc);
4798	return rc;
4799	}
4800
4801
4802	/**
4803	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4804	* hosts (except darwin) for 64-bit guests.
4805	*
4806	* @param pVCpu Pointer to the VMCPU.
4807	* @param idxField The VMCS field encoding.
4808	* @param u64Val 16, 32 or 64 bits value.
4809	*/
4810	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4811	{
4812	AssertPtr(pVCpu);
4813	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4814
4815	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4816	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4817
4818	/* Make sure there are no duplicates. */
4819	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4820	{
4821	if (pCache->Write.aField[i] == idxField)
4822	{
4823	pCache->Write.aFieldVal[i] = u64Val;
4824	return VINF_SUCCESS;
4825	}
4826	}
4827
4828	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4829	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4830	pCache->Write.cValidEntries++;
4831	return VINF_SUCCESS;
4832	}
4833
4834	/* Enable later when the assembly code uses these as callbacks. */
4835	#if 0
4836	/*
4837	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4838	*
4839	* @param pVCpu Pointer to the VMCPU.
4840	* @param pCache Pointer to the VMCS cache.
4841	*
4842	* @remarks No-long-jump zone!!!
4843	*/
4844	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4845	{
4846	AssertPtr(pCache);
4847	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4848	{
4849	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4850	AssertRC(rc);
4851	}
4852	pCache->Write.cValidEntries = 0;
4853	}
4854
4855
4856	/**
4857	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4858	*
4859	* @param pVCpu Pointer to the VMCPU.
4860	* @param pCache Pointer to the VMCS cache.
4861	*
4862	* @remarks No-long-jump zone!!!
4863	*/
4864	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4865	{
4866	AssertPtr(pCache);
4867	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4868	{
4869	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4870	AssertRC(rc);
4871	}
4872	}
4873	#endif
4874	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4875
4876
4877	/**
4878	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4879	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4880	* timer.
4881	*
4882	* @returns VBox status code.
4883	* @param pVCpu Pointer to the VMCPU.
4884	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4885	* out-of-sync. Make sure to update the required fields
4886	* before using them.
4887	* @remarks No-long-jump zone!!!
4888	*/
4889	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4890	{
4891	int rc = VERR_INTERNAL_ERROR_5;
4892	bool fOffsettedTsc = false;
4893	PVM pVM = pVCpu->CTX_SUFF(pVM);
4894	if (pVM->hm.s.vmx.fUsePreemptTimer)
4895	{
4896	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4897
4898	/* Make sure the returned values have sane upper and lower boundaries. */
4899	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4900	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4901	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4902	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4903
4904	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4905	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4906	}
4907	else
4908	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4909
4910	if (fOffsettedTsc)
4911	{
4912	uint64_t u64CurTSC = ASMReadTSC();
4913	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4914	{
4915	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4916	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4917
4918	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4919	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4920	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4921	}
4922	else
4923	{
4924	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4925	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4926	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4927	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4928	}
4929	}
4930	else
4931	{
4932	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4933	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4934	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4935	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4936	}
4937	}
4938
4939
4940	/**
4941	* Determines if an exception is a contributory exception. Contributory
4942	* exceptions are ones which can cause double-faults. Page-fault is
4943	* intentionally not included here as it's a conditional contributory exception.
4944	*
4945	* @returns true if the exception is contributory, false otherwise.
4946	* @param uVector The exception vector.
4947	*/
4948	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4949	{
4950	switch (uVector)
4951	{
4952	case X86_XCPT_GP:
4953	case X86_XCPT_SS:
4954	case X86_XCPT_NP:
4955	case X86_XCPT_TS:
4956	case X86_XCPT_DE:
4957	return true;
4958	default:
4959	break;
4960	}
4961	return false;
4962	}
4963
4964
4965	/**
4966	* Sets an event as a pending event to be injected into the guest.
4967	*
4968	* @param pVCpu Pointer to the VMCPU.
4969	* @param u32IntrInfo The VM-entry interruption-information field.
4970	* @param cbInstr The VM-entry instruction length in bytes (for software
4971	* interrupts, exceptions and privileged software
4972	* exceptions).
4973	* @param u32ErrCode The VM-entry exception error code.
4974	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4975	* page-fault.
4976	*
4977	* @remarks Statistics counter assumes this is a guest event being injected or
4978	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4979	* always incremented.
4980	*/
4981	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4982	RTGCUINTPTR GCPtrFaultAddress)
4983	{
4984	Assert(!pVCpu->hm.s.Event.fPending);
4985	pVCpu->hm.s.Event.fPending = true;
4986	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4987	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4988	pVCpu->hm.s.Event.cbInstr = cbInstr;
4989	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4990
4991	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4992	}
4993
4994
4995	/**
4996	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4997	*
4998	* @param pVCpu Pointer to the VMCPU.
4999	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5000	* out-of-sync. Make sure to update the required fields
5001	* before using them.
5002	*/
5003	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5004	{
5005	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5006	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5007	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5008	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5009	}
5010
5011
5012	/**
5013	* Handle a condition that occurred while delivering an event through the guest
5014	* IDT.
5015	*
5016	* @returns VBox status code (informational error codes included).
5017	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5018	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5019	* continue execution of the guest which will delivery the #DF.
5020	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5021	*
5022	* @param pVCpu Pointer to the VMCPU.
5023	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5024	* out-of-sync. Make sure to update the required fields
5025	* before using them.
5026	* @param pVmxTransient Pointer to the VMX transient structure.
5027	*
5028	* @remarks No-long-jump zone!!!
5029	*/
5030	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5031	{
5032	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5033	AssertRCReturn(rc, rc);
5034	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5035	{
5036	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
5037	AssertRCReturn(rc, rc);
5038
5039	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5040	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
5041	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5042
5043	typedef enum
5044	{
5045	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5046	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5047	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5048	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5049	} VMXREFLECTXCPT;
5050
5051	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5052	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5053	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
5054	{
5055	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5056	{
5057	enmReflect = VMXREFLECTXCPT_XCPT;
5058	#ifdef VBOX_STRICT
5059	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5060	&& uExitVector == X86_XCPT_PF)
5061	{
5062	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5063	}
5064	#endif
5065	if ( uExitVector == X86_XCPT_PF
5066	&& uIdtVector == X86_XCPT_PF)
5067	{
5068	pVmxTransient->fVectoringPF = true;
5069	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5070	}
5071	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5072	&& hmR0VmxIsContributoryXcpt(uExitVector)
5073	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5074	\|\| uIdtVector == X86_XCPT_PF))
5075	{
5076	enmReflect = VMXREFLECTXCPT_DF;
5077	}
5078	else if (uIdtVector == X86_XCPT_DF)
5079	enmReflect = VMXREFLECTXCPT_TF;
5080	}
5081	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5082	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5083	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5084	{
5085	/*
5086	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5087	* (whatever they are) as they reoccur when restarting the instruction.
5088	*/
5089	enmReflect = VMXREFLECTXCPT_XCPT;
5090	}
5091	}
5092	else
5093	{
5094	/*
5095	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5096	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5097	* original exception to the guest after handling the VM-exit.
5098	*/
5099	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5100	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5101	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5102	{
5103	enmReflect = VMXREFLECTXCPT_XCPT;
5104	}
5105	}
5106
5107	switch (enmReflect)
5108	{
5109	case VMXREFLECTXCPT_XCPT:
5110	{
5111	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5112	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5113	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5114
5115	uint32_t u32ErrCode = 0;
5116	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5117	{
5118	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5119	AssertRCReturn(rc, rc);
5120	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5121	}
5122
5123	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5124	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5125	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5126	rc = VINF_SUCCESS;
5127	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5128	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5129
5130	break;
5131	}
5132
5133	case VMXREFLECTXCPT_DF:
5134	{
5135	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5136	rc = VINF_HM_DOUBLE_FAULT;
5137	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5138	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5139
5140	break;
5141	}
5142
5143	case VMXREFLECTXCPT_TF:
5144	{
5145	rc = VINF_EM_RESET;
5146	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5147	uExitVector));
5148	break;
5149	}
5150
5151	default:
5152	Assert(rc == VINF_SUCCESS);
5153	break;
5154	}
5155	}
5156	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5157	return rc;
5158	}
5159
5160
5161	/**
5162	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5163	*
5164	* @returns VBox status code.
5165	* @param pVCpu Pointer to the VMCPU.
5166	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5167	* out-of-sync. Make sure to update the required fields
5168	* before using them.
5169	*
5170	* @remarks No-long-jump zone!!!
5171	*/
5172	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5173	{
5174	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5175	{
5176	uint32_t uVal = 0;
5177	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5178	AssertRCReturn(rc, rc);
5179
5180	uint32_t uShadow = 0;
5181	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5182	AssertRCReturn(rc, rc);
5183
5184	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5185	CPUMSetGuestCR0(pVCpu, uVal);
5186	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5187	}
5188	return VINF_SUCCESS;
5189	}
5190
5191
5192	/**
5193	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5194	*
5195	* @returns VBox status code.
5196	* @param pVCpu Pointer to the VMCPU.
5197	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5198	* out-of-sync. Make sure to update the required fields
5199	* before using them.
5200	*
5201	* @remarks No-long-jump zone!!!
5202	*/
5203	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5204	{
5205	int rc = VINF_SUCCESS;
5206	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5207	{
5208	uint32_t uVal = 0;
5209	uint32_t uShadow = 0;
5210	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5211	AssertRCReturn(rc, rc);
5212	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5213	AssertRCReturn(rc, rc);
5214
5215	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5216	CPUMSetGuestCR4(pVCpu, uVal);
5217	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5218	}
5219	return rc;
5220	}
5221
5222
5223	/**
5224	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5225	*
5226	* @returns VBox status code.
5227	* @param pVCpu Pointer to the VMCPU.
5228	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5229	* out-of-sync. Make sure to update the required fields
5230	* before using them.
5231	*
5232	* @remarks No-long-jump zone!!!
5233	*/
5234	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5235	{
5236	int rc = VINF_SUCCESS;
5237	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5238	{
5239	uint64_t u64Val = 0;
5240	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5241	AssertRCReturn(rc, rc);
5242
5243	pMixedCtx->rip = u64Val;
5244	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5245	}
5246	return rc;
5247	}
5248
5249
5250	/**
5251	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5252	*
5253	* @returns VBox status code.
5254	* @param pVCpu Pointer to the VMCPU.
5255	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5256	* out-of-sync. Make sure to update the required fields
5257	* before using them.
5258	*
5259	* @remarks No-long-jump zone!!!
5260	*/
5261	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5262	{
5263	int rc = VINF_SUCCESS;
5264	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5265	{
5266	uint64_t u64Val = 0;
5267	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5268	AssertRCReturn(rc, rc);
5269
5270	pMixedCtx->rsp = u64Val;
5271	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5272	}
5273	return rc;
5274	}
5275
5276
5277	/**
5278	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5279	*
5280	* @returns VBox status code.
5281	* @param pVCpu Pointer to the VMCPU.
5282	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5283	* out-of-sync. Make sure to update the required fields
5284	* before using them.
5285	*
5286	* @remarks No-long-jump zone!!!
5287	*/
5288	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5289	{
5290	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5291	{
5292	uint32_t uVal = 0;
5293	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5294	AssertRCReturn(rc, rc);
5295
5296	pMixedCtx->eflags.u32 = uVal;
5297	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5298	{
5299	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5300	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5301
5302	pMixedCtx->eflags.Bits.u1VM = 0;
5303	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5304	}
5305
5306	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5307	}
5308	return VINF_SUCCESS;
5309	}
5310
5311
5312	/**
5313	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5314	* guest-CPU context.
5315	*/
5316	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5317	{
5318	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5319	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5320	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5321	return rc;
5322	}
5323
5324
5325	/**
5326	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5327	* from the guest-state area in the VMCS.
5328	*
5329	* @param pVCpu Pointer to the VMCPU.
5330	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5331	* out-of-sync. Make sure to update the required fields
5332	* before using them.
5333	*
5334	* @remarks No-long-jump zone!!!
5335	*/
5336	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5337	{
5338	uint32_t uIntrState = 0;
5339	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5340	AssertRC(rc);
5341
5342	if (!uIntrState)
5343	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5344	else
5345	{
5346	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5347	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5348	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5349	AssertRC(rc);
5350	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5351	AssertRC(rc);
5352
5353	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5354	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5355	}
5356	}
5357
5358
5359	/**
5360	* Saves the guest's activity state.
5361	*
5362	* @returns VBox status code.
5363	* @param pVCpu Pointer to the VMCPU.
5364	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5365	* out-of-sync. Make sure to update the required fields
5366	* before using them.
5367	*
5368	* @remarks No-long-jump zone!!!
5369	*/
5370	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5371	{
5372	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5373	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5374	return VINF_SUCCESS;
5375	}
5376
5377
5378	/**
5379	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5380	* the current VMCS into the guest-CPU context.
5381	*
5382	* @returns VBox status code.
5383	* @param pVCpu Pointer to the VMCPU.
5384	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5385	* out-of-sync. Make sure to update the required fields
5386	* before using them.
5387	*
5388	* @remarks No-long-jump zone!!!
5389	*/
5390	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5391	{
5392	int rc = VINF_SUCCESS;
5393	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5394	{
5395	uint32_t u32Val = 0;
5396	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5397	pMixedCtx->SysEnter.cs = u32Val;
5398	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5399	}
5400
5401	uint64_t u64Val = 0;
5402	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5403	{
5404	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5405	pMixedCtx->SysEnter.eip = u64Val;
5406	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5407	}
5408	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5409	{
5410	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5411	pMixedCtx->SysEnter.esp = u64Val;
5412	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5413	}
5414	return rc;
5415	}
5416
5417
5418	/**
5419	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5420	* context.
5421	*
5422	* @returns VBox status code.
5423	* @param pVCpu Pointer to the VMCPU.
5424	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5425	* out-of-sync. Make sure to update the required fields
5426	* before using them.
5427	*
5428	* @remarks No-long-jump zone!!!
5429	*/
5430	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5431	{
5432	int rc = VINF_SUCCESS;
5433	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5434	{
5435	uint64_t u64Val = 0;
5436	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5437	pMixedCtx->fs.u64Base = u64Val;
5438	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5439	}
5440	return rc;
5441	}
5442
5443
5444	/**
5445	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5446	* context.
5447	*
5448	* @returns VBox status code.
5449	* @param pVCpu Pointer to the VMCPU.
5450	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5451	* out-of-sync. Make sure to update the required fields
5452	* before using them.
5453	*
5454	* @remarks No-long-jump zone!!!
5455	*/
5456	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5457	{
5458	int rc = VINF_SUCCESS;
5459	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5460	{
5461	uint64_t u64Val = 0;
5462	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5463	pMixedCtx->gs.u64Base = u64Val;
5464	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5465	}
5466	return rc;
5467	}
5468
5469
5470	/**
5471	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5472	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5473	* and TSC_AUX.
5474	*
5475	* @returns VBox status code.
5476	* @param pVCpu Pointer to the VMCPU.
5477	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5478	* out-of-sync. Make sure to update the required fields
5479	* before using them.
5480	*
5481	* @remarks No-long-jump zone!!!
5482	*/
5483	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5484	{
5485	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5486	return VINF_SUCCESS;
5487
5488	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5489	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5490	{
5491	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5492	pMsr += i;
5493	switch (pMsr->u32Msr)
5494	{
5495	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5496	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5497	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5498	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5499	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5500	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5501	default:
5502	{
5503	AssertFailed();
5504	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5505	}
5506	}
5507	}
5508	#endif
5509
5510	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5511	return VINF_SUCCESS;
5512	}
5513
5514
5515	/**
5516	* Saves the guest control registers from the current VMCS into the guest-CPU
5517	* context.
5518	*
5519	* @returns VBox status code.
5520	* @param pVCpu Pointer to the VMCPU.
5521	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5522	* out-of-sync. Make sure to update the required fields
5523	* before using them.
5524	*
5525	* @remarks No-long-jump zone!!!
5526	*/
5527	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5528	{
5529	/* Guest CR0. Guest FPU. */
5530	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5531	AssertRCReturn(rc, rc);
5532
5533	/* Guest CR4. */
5534	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5535	AssertRCReturn(rc, rc);
5536
5537	/* Guest CR2 - updated always during the world-switch or in #PF. */
5538	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5539	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5540	{
5541	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5542	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5543
5544	PVM pVM = pVCpu->CTX_SUFF(pVM);
5545	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5546	\|\| ( pVM->hm.s.fNestedPaging
5547	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5548	{
5549	uint64_t u64Val = 0;
5550	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5551	if (pMixedCtx->cr3 != u64Val)
5552	{
5553	CPUMSetGuestCR3(pVCpu, u64Val);
5554	if (VMMRZCallRing3IsEnabled(pVCpu))
5555	{
5556	PGMUpdateCR3(pVCpu, u64Val);
5557	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5558	}
5559	else
5560	{
5561	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5562	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5563	}
5564	}
5565
5566	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5567	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5568	{
5569	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5570	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5571	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5572	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5573
5574	if (VMMRZCallRing3IsEnabled(pVCpu))
5575	{
5576	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5577	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5578	}
5579	else
5580	{
5581	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5582	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5583	}
5584	}
5585	}
5586
5587	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5588	}
5589
5590	/*
5591	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5592	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5593	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5594	*
5595	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5596	*/
5597	if (VMMRZCallRing3IsEnabled(pVCpu))
5598	{
5599	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5600	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5601
5602	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5603	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5604
5605	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5606	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5607	}
5608
5609	return rc;
5610	}
5611
5612
5613	/**
5614	* Reads a guest segment register from the current VMCS into the guest-CPU
5615	* context.
5616	*
5617	* @returns VBox status code.
5618	* @param pVCpu Pointer to the VMCPU.
5619	* @param idxSel Index of the selector in the VMCS.
5620	* @param idxLimit Index of the segment limit in the VMCS.
5621	* @param idxBase Index of the segment base in the VMCS.
5622	* @param idxAccess Index of the access rights of the segment in the VMCS.
5623	* @param pSelReg Pointer to the segment selector.
5624	*
5625	* @remarks No-long-jump zone!!!
5626	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5627	* macro as that takes care of whether to read from the VMCS cache or
5628	* not.
5629	*/
5630	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5631	PCPUMSELREG pSelReg)
5632	{
5633	uint32_t u32Val = 0;
5634	int rc = VMXReadVmcs32(idxSel, &u32Val);
5635	AssertRCReturn(rc, rc);
5636	pSelReg->Sel = (uint16_t)u32Val;
5637	pSelReg->ValidSel = (uint16_t)u32Val;
5638	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5639
5640	rc = VMXReadVmcs32(idxLimit, &u32Val);
5641	AssertRCReturn(rc, rc);
5642	pSelReg->u32Limit = u32Val;
5643
5644	uint64_t u64Val = 0;
5645	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5646	AssertRCReturn(rc, rc);
5647	pSelReg->u64Base = u64Val;
5648
5649	rc = VMXReadVmcs32(idxAccess, &u32Val);
5650	AssertRCReturn(rc, rc);
5651	pSelReg->Attr.u = u32Val;
5652
5653	/*
5654	* If VT-x marks the segment as unusable, most other bits remain undefined:
5655	* - For CS the L, D and G bits have meaning.
5656	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5657	* - For the remaining data segments no bits are defined.
5658	*
5659	* The present bit and the unusable bit has been observed to be set at the
5660	* same time (the selector was supposed to invalid as we started executing
5661	* a V8086 interrupt in ring-0).
5662	*
5663	* What should be important for the rest of the VBox code that the P bit is
5664	* cleared. Some of the other VBox code recognizes the unusable bit, but
5665	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5666	* safe side here, we'll strip off P and other bits we don't care about. If
5667	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5668	*
5669	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5670	*/
5671	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5672	{
5673	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5674
5675	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5676	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5677	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5678
5679	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5680	#ifdef DEBUG_bird
5681	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5682	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5683	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5684	#endif
5685	}
5686	return VINF_SUCCESS;
5687	}
5688
5689
5690	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5691	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5692	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5693	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5694	#else
5695	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5696	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5697	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5698	#endif
5699
5700
5701	/**
5702	* Saves the guest segment registers from the current VMCS into the guest-CPU
5703	* context.
5704	*
5705	* @returns VBox status code.
5706	* @param pVCpu Pointer to the VMCPU.
5707	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5708	* out-of-sync. Make sure to update the required fields
5709	* before using them.
5710	*
5711	* @remarks No-long-jump zone!!!
5712	*/
5713	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5714	{
5715	/* Guest segment registers. */
5716	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5717	{
5718	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5719	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5720	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5721	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5722	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5723	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5724	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5725
5726	/* Restore segment attributes for real-on-v86 mode hack. */
5727	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5728	{
5729	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5730	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5731	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5732	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5733	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5734	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5735	}
5736	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5737	}
5738
5739	return VINF_SUCCESS;
5740	}
5741
5742
5743	/**
5744	* Saves the guest descriptor table registers and task register from the current
5745	* VMCS into the guest-CPU context.
5746	*
5747	* @returns VBox status code.
5748	* @param pVCpu Pointer to the VMCPU.
5749	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5750	* out-of-sync. Make sure to update the required fields
5751	* before using them.
5752	*
5753	* @remarks No-long-jump zone!!!
5754	*/
5755	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5756	{
5757	int rc = VINF_SUCCESS;
5758
5759	/* Guest LDTR. */
5760	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5761	{
5762	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5763	AssertRCReturn(rc, rc);
5764	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5765	}
5766
5767	/* Guest GDTR. */
5768	uint64_t u64Val = 0;
5769	uint32_t u32Val = 0;
5770	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5771	{
5772	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5773	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5774	pMixedCtx->gdtr.pGdt = u64Val;
5775	pMixedCtx->gdtr.cbGdt = u32Val;
5776	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5777	}
5778
5779	/* Guest IDTR. */
5780	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5781	{
5782	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5783	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5784	pMixedCtx->idtr.pIdt = u64Val;
5785	pMixedCtx->idtr.cbIdt = u32Val;
5786	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5787	}
5788
5789	/* Guest TR. */
5790	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5791	{
5792	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5793	AssertRCReturn(rc, rc);
5794
5795	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5796	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5797	{
5798	rc = VMXLOCAL_READ_SEG(TR, tr);
5799	AssertRCReturn(rc, rc);
5800	}
5801	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5802	}
5803	return rc;
5804	}
5805
5806	#undef VMXLOCAL_READ_SEG
5807
5808
5809	/**
5810	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5811	* context.
5812	*
5813	* @returns VBox status code.
5814	* @param pVCpu Pointer to the VMCPU.
5815	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5816	* out-of-sync. Make sure to update the required fields
5817	* before using them.
5818	*
5819	* @remarks No-long-jump zone!!!
5820	*/
5821	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5822	{
5823	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5824	{
5825	if (!pVCpu->hm.s.fUsingHyperDR7)
5826	{
5827	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5828	uint32_t u32Val;
5829	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5830	pMixedCtx->dr[7] = u32Val;
5831	}
5832
5833	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5834	}
5835	return VINF_SUCCESS;
5836	}
5837
5838
5839	/**
5840	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5841	*
5842	* @returns VBox status code.
5843	* @param pVCpu Pointer to the VMCPU.
5844	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5845	* out-of-sync. Make sure to update the required fields
5846	* before using them.
5847	*
5848	* @remarks No-long-jump zone!!!
5849	*/
5850	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5851	{
5852	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5853	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5854	return VINF_SUCCESS;
5855	}
5856
5857
5858	/**
5859	* Saves the entire guest state from the currently active VMCS into the
5860	* guest-CPU context. This essentially VMREADs all guest-data.
5861	*
5862	* @returns VBox status code.
5863	* @param pVCpu Pointer to the VMCPU.
5864	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5865	* out-of-sync. Make sure to update the required fields
5866	* before using them.
5867	*/
5868	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5869	{
5870	Assert(pVCpu);
5871	Assert(pMixedCtx);
5872
5873	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5874	return VINF_SUCCESS;
5875
5876	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5877	again on the ring-3 callback path, there is no real need to. */
5878	if (VMMRZCallRing3IsEnabled(pVCpu))
5879	VMMR0LogFlushDisable(pVCpu);
5880	else
5881	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5882	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5883
5884	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5885	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5886
5887	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5888	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5889
5890	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5891	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5892
5893	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5894	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5895
5896	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5897	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5898
5899	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5900	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5901
5902	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5903	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5904
5905	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5906	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5907
5908	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5909	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5910
5911	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5912	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5913
5914	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5915	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5916
5917	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5918	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5919
5920	if (VMMRZCallRing3IsEnabled(pVCpu))
5921	VMMR0LogFlushEnable(pVCpu);
5922
5923	return rc;
5924	}
5925
5926
5927	/**
5928	* Check per-VM and per-VCPU force flag actions that require us to go back to
5929	* ring-3 for one reason or another.
5930	*
5931	* @returns VBox status code (information status code included).
5932	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5933	* ring-3.
5934	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5935	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5936	* interrupts)
5937	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5938	* all EMTs to be in ring-3.
5939	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5940	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5941	* to the EM loop.
5942	*
5943	* @param pVM Pointer to the VM.
5944	* @param pVCpu Pointer to the VMCPU.
5945	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5946	* out-of-sync. Make sure to update the required fields
5947	* before using them.
5948	*/
5949	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5950	{
5951	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5952
5953	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5954	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5955	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5956	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5957	{
5958	/* We need the control registers now, make sure the guest-CPU context is updated. */
5959	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5960	AssertRCReturn(rc3, rc3);
5961
5962	/* Pending HM CR3 sync. */
5963	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5964	{
5965	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5966	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
5967	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5968	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5969	}
5970
5971	/* Pending HM PAE PDPEs. */
5972	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5973	{
5974	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5975	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5976	}
5977
5978	/* Pending PGM C3 sync. */
5979	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5980	{
5981	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
5982	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5983	if (rc2 != VINF_SUCCESS)
5984	{
5985	AssertRC(rc2);
5986	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5987	return rc2;
5988	}
5989	}
5990
5991	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5992	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5993	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5994	{
5995	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5996	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5997	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5998	return rc2;
5999	}
6000
6001	/* Pending VM request packets, such as hardware interrupts. */
6002	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6003	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6004	{
6005	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6006	return VINF_EM_PENDING_REQUEST;
6007	}
6008
6009	/* Pending PGM pool flushes. */
6010	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6011	{
6012	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6013	return VINF_PGM_POOL_FLUSH_PENDING;
6014	}
6015
6016	/* Pending DMA requests. */
6017	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6018	{
6019	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6020	return VINF_EM_RAW_TO_R3;
6021	}
6022	}
6023
6024	return VINF_SUCCESS;
6025	}
6026
6027
6028	/**
6029	* Converts any TRPM trap into a pending HM event. This is typically used when
6030	* entering from ring-3 (not longjmp returns).
6031	*
6032	* @param pVCpu Pointer to the VMCPU.
6033	*/
6034	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6035	{
6036	Assert(TRPMHasTrap(pVCpu));
6037	Assert(!pVCpu->hm.s.Event.fPending);
6038
6039	uint8_t uVector;
6040	TRPMEVENT enmTrpmEvent;
6041	RTGCUINT uErrCode;
6042	RTGCUINTPTR GCPtrFaultAddress;
6043	uint8_t cbInstr;
6044
6045	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6046	AssertRC(rc);
6047
6048	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
6049	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6050	if (enmTrpmEvent == TRPM_TRAP)
6051	{
6052	switch (uVector)
6053	{
6054	case X86_XCPT_BP:
6055	case X86_XCPT_OF:
6056	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6057	break;
6058
6059	case X86_XCPT_PF:
6060	case X86_XCPT_DF:
6061	case X86_XCPT_TS:
6062	case X86_XCPT_NP:
6063	case X86_XCPT_SS:
6064	case X86_XCPT_GP:
6065	case X86_XCPT_AC:
6066	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6067	/* no break! */
6068	default:
6069	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6070	break;
6071	}
6072	}
6073	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6074	{
6075	if (uVector == X86_XCPT_NMI)
6076	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6077	else
6078	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6079	}
6080	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6081	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6082	else
6083	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6084
6085	rc = TRPMResetTrap(pVCpu);
6086	AssertRC(rc);
6087	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6088	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6089
6090	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6091	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6092	}
6093
6094
6095	/**
6096	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6097	* VT-x to execute any instruction.
6098	*
6099	* @param pvCpu Pointer to the VMCPU.
6100	*/
6101	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6102	{
6103	Assert(pVCpu->hm.s.Event.fPending);
6104
6105	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6106	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6107	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6108	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6109
6110	/* If a trap was already pending, we did something wrong! */
6111	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6112
6113	TRPMEVENT enmTrapType;
6114	switch (uVectorType)
6115	{
6116	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6117	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6118	enmTrapType = TRPM_HARDWARE_INT;
6119	break;
6120
6121	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6122	enmTrapType = TRPM_SOFTWARE_INT;
6123	break;
6124
6125	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6126	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6127	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6128	enmTrapType = TRPM_TRAP;
6129	break;
6130
6131	default:
6132	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6133	enmTrapType = TRPM_32BIT_HACK;
6134	break;
6135	}
6136
6137	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6138
6139	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6140	AssertRC(rc);
6141
6142	if (fErrorCodeValid)
6143	TRPMSetErrorCode(pVCpu, uErrorCode);
6144
6145	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6146	&& uVector == X86_XCPT_PF)
6147	{
6148	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6149	}
6150	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6151	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6152	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6153	{
6154	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6155	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6156	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6157	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6158	}
6159	pVCpu->hm.s.Event.fPending = false;
6160	}
6161
6162
6163	/**
6164	* Does the necessary state syncing before returning to ring-3 for any reason
6165	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6166	*
6167	* @returns VBox status code.
6168	* @param pVM Pointer to the VM.
6169	* @param pVCpu Pointer to the VMCPU.
6170	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6171	* be out-of-sync. Make sure to update the required
6172	* fields before using them.
6173	* @param fSaveGuestState Whether to save the guest state or not.
6174	*
6175	* @remarks No-long-jmp zone!!!
6176	*/
6177	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6178	{
6179	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6180	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6181
6182	RTCPUID idCpu = RTMpCpuId();
6183	Log4Func(("HostCpuId=%u\n", idCpu));
6184
6185	/* Save the guest state if necessary. */
6186	if ( fSaveGuestState
6187	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6188	{
6189	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6190	AssertRCReturn(rc, rc);
6191	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6192	}
6193
6194	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6195	if (CPUMIsGuestFPUStateActive(pVCpu))
6196	{
6197	/* We shouldn't reload CR0 without saving it first. */
6198	if (!fSaveGuestState)
6199	{
6200	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6201	AssertRCReturn(rc, rc);
6202	}
6203	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6204	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6205	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6206	}
6207
6208	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6209	#ifdef VBOX_STRICT
6210	if (CPUMIsHyperDebugStateActive(pVCpu))
6211	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6212	#endif
6213	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6214	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6215	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6216	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6217
6218	#if HC_ARCH_BITS == 64
6219	/* Restore host-state bits that VT-x only restores partially. */
6220	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6221	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6222	{
6223	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6224	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6225	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6226	}
6227	#endif
6228
6229	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6230	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6231	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6232	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6233	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6234	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6235	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6236	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6237
6238	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6239
6240	/** @todo This kinda defeats the purpose of having preemption hooks.
6241	* The problem is, deregistering the hooks should be moved to a place that
6242	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6243	* context.
6244	*/
6245	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6246	{
6247	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6248	AssertRCReturn(rc, rc);
6249
6250	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6251	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6252	}
6253	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6254	NOREF(idCpu);
6255
6256	return VINF_SUCCESS;
6257	}
6258
6259
6260	/**
6261	* Leaves the VT-x session.
6262	*
6263	* @returns VBox status code.
6264	* @param pVM Pointer to the VM.
6265	* @param pVCpu Pointer to the VMCPU.
6266	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6267	* out-of-sync. Make sure to update the required fields
6268	* before using them.
6269	*
6270	* @remarks No-long-jmp zone!!!
6271	*/
6272	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6273	{
6274	HM_DISABLE_PREEMPT_IF_NEEDED();
6275	HMVMX_ASSERT_CPU_SAFE();
6276	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6277	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6278
6279	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6280	and done this from the VMXR0ThreadCtxCallback(). */
6281	if (!pVCpu->hm.s.fLeaveDone)
6282	{
6283	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6284	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6285	pVCpu->hm.s.fLeaveDone = true;
6286	}
6287
6288	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6289	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6290	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6291	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6292	VMMR0ThreadCtxHooksDeregister(pVCpu);
6293
6294	/* Leave HM context. This takes care of local init (term). */
6295	int rc = HMR0LeaveCpu(pVCpu);
6296
6297	HM_RESTORE_PREEMPT_IF_NEEDED();
6298
6299	return rc;
6300	}
6301
6302
6303	/**
6304	* Does the necessary state syncing before doing a longjmp to ring-3.
6305	*
6306	* @returns VBox status code.
6307	* @param pVM Pointer to the VM.
6308	* @param pVCpu Pointer to the VMCPU.
6309	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6310	* out-of-sync. Make sure to update the required fields
6311	* before using them.
6312	*
6313	* @remarks No-long-jmp zone!!!
6314	*/
6315	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6316	{
6317	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6318	}
6319
6320
6321	/**
6322	* Take necessary actions before going back to ring-3.
6323	*
6324	* An action requires us to go back to ring-3. This function does the necessary
6325	* steps before we can safely return to ring-3. This is not the same as longjmps
6326	* to ring-3, this is voluntary and prepares the guest so it may continue
6327	* executing outside HM (recompiler/IEM).
6328	*
6329	* @returns VBox status code.
6330	* @param pVM Pointer to the VM.
6331	* @param pVCpu Pointer to the VMCPU.
6332	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6333	* out-of-sync. Make sure to update the required fields
6334	* before using them.
6335	* @param rcExit The reason for exiting to ring-3. Can be
6336	* VINF_VMM_UNKNOWN_RING3_CALL.
6337	*/
6338	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6339	{
6340	Assert(pVM);
6341	Assert(pVCpu);
6342	Assert(pMixedCtx);
6343	HMVMX_ASSERT_PREEMPT_SAFE();
6344
6345	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6346	{
6347	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6348	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6349	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6350	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6351	}
6352
6353	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6354	VMMRZCallRing3Disable(pVCpu);
6355	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6356
6357	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6358	if (pVCpu->hm.s.Event.fPending)
6359	{
6360	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6361	Assert(!pVCpu->hm.s.Event.fPending);
6362	}
6363
6364	/* Save guest state and restore host state bits. */
6365	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6366	AssertRCReturn(rc, rc);
6367	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6368
6369	/* Sync recompiler state. */
6370	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6371	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6372	\| CPUM_CHANGED_LDTR
6373	\| CPUM_CHANGED_GDTR
6374	\| CPUM_CHANGED_IDTR
6375	\| CPUM_CHANGED_TR
6376	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6377	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6378	if ( pVM->hm.s.fNestedPaging
6379	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6380	{
6381	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6382	}
6383
6384	/*
6385	* Clear the X86_EFL_TF if necessary.
6386	*/
6387	if (pVCpu->hm.s.fClearTrapFlag)
6388	{
6389	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6390	pMixedCtx->eflags.Bits.u1TF = 0;
6391	pVCpu->hm.s.fClearTrapFlag = false;
6392	}
6393	/** @todo there seems to be issues with the resume flag when the monitor trap
6394	* flag is pending without being used. Seen early in bios init when
6395	* accessing APIC page in prot mode. */
6396
6397	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6398	if (rcExit != VINF_EM_RAW_INTERRUPT)
6399	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6400
6401	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6402
6403	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6404	VMMRZCallRing3RemoveNotification(pVCpu);
6405	VMMRZCallRing3Enable(pVCpu);
6406
6407	return rc;
6408	}
6409
6410
6411	/**
6412	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6413	* longjump to ring-3 and possibly get preempted.
6414	*
6415	* @returns VBox status code.
6416	* @param pVCpu Pointer to the VMCPU.
6417	* @param enmOperation The operation causing the ring-3 longjump.
6418	* @param pvUser Opaque pointer to the guest-CPU context. The data
6419	* may be out-of-sync. Make sure to update the required
6420	* fields before using them.
6421	*/
6422	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6423	{
6424	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6425	{
6426	VMMRZCallRing3RemoveNotification(pVCpu);
6427	HM_DISABLE_PREEMPT_IF_NEEDED();
6428
6429	/* If anything here asserts or fails, good luck. */
6430	if (CPUMIsGuestFPUStateActive(pVCpu))
6431	CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6432
6433	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6434
6435	#if HC_ARCH_BITS == 64
6436	/* Restore host-state bits that VT-x only restores partially. */
6437	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6438	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6439	{
6440	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6441	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6442	}
6443	#endif
6444	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6445	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6446	{
6447	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6448	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6449	}
6450
6451	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6452	VMMR0ThreadCtxHooksDeregister(pVCpu);
6453
6454	HMR0LeaveCpu(pVCpu);
6455	HM_RESTORE_PREEMPT_IF_NEEDED();
6456	return VINF_SUCCESS;
6457	}
6458
6459	Assert(pVCpu);
6460	Assert(pvUser);
6461	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6462	HMVMX_ASSERT_PREEMPT_SAFE();
6463
6464	VMMRZCallRing3Disable(pVCpu);
6465	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6466
6467	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6468	enmOperation));
6469
6470	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6471	AssertRCReturn(rc, rc);
6472
6473	VMMRZCallRing3Enable(pVCpu);
6474	return VINF_SUCCESS;
6475	}
6476
6477
6478	/**
6479	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6480	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6481	*
6482	* @param pVCpu Pointer to the VMCPU.
6483	*/
6484	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6485	{
6486	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6487	{
6488	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6489	{
6490	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6491	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6492	AssertRC(rc);
6493	}
6494	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6495	}
6496
6497
6498	/**
6499	* Evaluates the event to be delivered to the guest and sets it as the pending
6500	* event.
6501	*
6502	* @param pVCpu Pointer to the VMCPU.
6503	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6504	* out-of-sync. Make sure to update the required fields
6505	* before using them.
6506	*/
6507	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6508	{
6509	Assert(!pVCpu->hm.s.Event.fPending);
6510
6511	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6512	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6513	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6514	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6515
6516	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6517	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6518	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6519	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6520	Assert(!TRPMHasTrap(pVCpu));
6521
6522	/** @todo SMI. SMIs take priority over NMIs. */
6523	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6524	{
6525	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6526	if ( !fBlockMovSS
6527	&& !fBlockSti)
6528	{
6529	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6530	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6531	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6532	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6533
6534	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6535	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6536	}
6537	else
6538	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6539	}
6540	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6541	&& !pVCpu->hm.s.fSingleInstruction)
6542	{
6543	/*
6544	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6545	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6546	* evaluated here and not set as pending, solely based on the force-flags.
6547	*/
6548	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6549	AssertRC(rc);
6550	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6551	if ( !fBlockInt
6552	&& !fBlockSti
6553	&& !fBlockMovSS)
6554	{
6555	uint8_t u8Interrupt;
6556	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6557	if (RT_SUCCESS(rc))
6558	{
6559	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6560	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6561	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6562
6563	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6564	}
6565	else
6566	{
6567	/** @todo Does this actually happen? If not turn it into an assertion. */
6568	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6569	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6570	}
6571	}
6572	else
6573	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6574	}
6575	}
6576
6577
6578	/**
6579	* Injects any pending events into the guest if the guest is in a state to
6580	* receive them.
6581	*
6582	* @returns VBox status code (informational status codes included).
6583	* @param pVCpu Pointer to the VMCPU.
6584	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6585	* out-of-sync. Make sure to update the required fields
6586	* before using them.
6587	*/
6588	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6589	{
6590	HMVMX_ASSERT_PREEMPT_SAFE();
6591	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6592
6593	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6594	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6595	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6596	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6597
6598	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6599	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6600	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6601	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6602	Assert(!TRPMHasTrap(pVCpu));
6603
6604	int rc = VINF_SUCCESS;
6605	if (pVCpu->hm.s.Event.fPending)
6606	{
6607	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
6608	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6609	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6610	{
6611	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6612	AssertRCReturn(rc, rc);
6613	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6614	Assert(!fBlockInt);
6615	Assert(!fBlockSti);
6616	Assert(!fBlockMovSS);
6617	}
6618	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6619	{
6620	Assert(!fBlockSti);
6621	Assert(!fBlockMovSS);
6622	}
6623	#endif
6624	Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6625	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6626	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6627	AssertRCReturn(rc, rc);
6628
6629	/* Update the interruptibility-state as it could have been changed by
6630	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6631	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6632	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6633
6634	#ifdef VBOX_WITH_STATISTICS
6635	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6636	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6637	else
6638	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6639	#endif
6640	}
6641
6642	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6643	int rc2 = VINF_SUCCESS;
6644	if ( fBlockSti
6645	\|\| fBlockMovSS)
6646	{
6647	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6648	{
6649	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6650	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6651	{
6652	/*
6653	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6654	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6655	* See Intel spec. 27.3.4 "Saving Non-Register State".
6656	*/
6657	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6658	AssertRCReturn(rc, rc);
6659	}
6660	}
6661	else
6662	{
6663	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6664	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6665	uIntrState = 0;
6666	}
6667	}
6668
6669	/*
6670	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6671	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6672	*/
6673	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6674	AssertRC(rc2);
6675
6676	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6677	return rc;
6678	}
6679
6680
6681	/**
6682	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6683	*
6684	* @param pVCpu Pointer to the VMCPU.
6685	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6686	* out-of-sync. Make sure to update the required fields
6687	* before using them.
6688	*/
6689	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6690	{
6691	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6692	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6693	}
6694
6695
6696	/**
6697	* Injects a double-fault (#DF) exception into the VM.
6698	*
6699	* @returns VBox status code (informational status code included).
6700	* @param pVCpu Pointer to the VMCPU.
6701	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6702	* out-of-sync. Make sure to update the required fields
6703	* before using them.
6704	*/
6705	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6706	{
6707	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6708	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6709	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6710	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6711	puIntrState);
6712	}
6713
6714
6715	/**
6716	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6717	*
6718	* @param pVCpu Pointer to the VMCPU.
6719	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6720	* out-of-sync. Make sure to update the required fields
6721	* before using them.
6722	*/
6723	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6724	{
6725	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6726	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6727	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6728	}
6729
6730
6731	/**
6732	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6733	*
6734	* @param pVCpu Pointer to the VMCPU.
6735	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6736	* out-of-sync. Make sure to update the required fields
6737	* before using them.
6738	* @param cbInstr The value of RIP that is to be pushed on the guest
6739	* stack.
6740	*/
6741	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6742	{
6743	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6744	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6745	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6746	}
6747
6748
6749	/**
6750	* Injects a general-protection (#GP) fault into the VM.
6751	*
6752	* @returns VBox status code (informational status code included).
6753	* @param pVCpu Pointer to the VMCPU.
6754	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6755	* out-of-sync. Make sure to update the required fields
6756	* before using them.
6757	* @param u32ErrorCode The error code associated with the #GP.
6758	*/
6759	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6760	uint32_t *puIntrState)
6761	{
6762	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6763	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6764	if (fErrorCodeValid)
6765	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6766	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6767	puIntrState);
6768	}
6769
6770
6771	/**
6772	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6773	*
6774	* @param pVCpu Pointer to the VMCPU.
6775	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6776	* out-of-sync. Make sure to update the required fields
6777	* before using them.
6778	* @param uVector The software interrupt vector number.
6779	* @param cbInstr The value of RIP that is to be pushed on the guest
6780	* stack.
6781	*/
6782	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6783	{
6784	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6785	if ( uVector == X86_XCPT_BP
6786	\|\| uVector == X86_XCPT_OF)
6787	{
6788	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6789	}
6790	else
6791	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6792	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6793	}
6794
6795
6796	/**
6797	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6798	* stack.
6799	*
6800	* @returns VBox status code (information status code included).
6801	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6802	* @param pVM Pointer to the VM.
6803	* @param pMixedCtx Pointer to the guest-CPU context.
6804	* @param uValue The value to push to the guest stack.
6805	*/
6806	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6807	{
6808	/*
6809	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6810	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6811	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6812	*/
6813	if (pMixedCtx->sp == 1)
6814	return VINF_EM_RESET;
6815	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6816	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6817	AssertRCReturn(rc, rc);
6818	return rc;
6819	}
6820
6821
6822	/**
6823	* Injects an event into the guest upon VM-entry by updating the relevant fields
6824	* in the VM-entry area in the VMCS.
6825	*
6826	* @returns VBox status code (informational error codes included).
6827	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6828	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6829	*
6830	* @param pVCpu Pointer to the VMCPU.
6831	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6832	* be out-of-sync. Make sure to update the required
6833	* fields before using them.
6834	* @param u64IntrInfo The VM-entry interruption-information field.
6835	* @param cbInstr The VM-entry instruction length in bytes (for
6836	* software interrupts, exceptions and privileged
6837	* software exceptions).
6838	* @param u32ErrCode The VM-entry exception error code.
6839	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6840	* @param puIntrState Pointer to the current guest interruptibility-state.
6841	* This interruptibility-state will be updated if
6842	* necessary. This cannot not be NULL.
6843	*
6844	* @remarks Requires CR0!
6845	* @remarks No-long-jump zone!!!
6846	*/
6847	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6848	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6849	{
6850	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6851	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6852	Assert(puIntrState);
6853	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6854
6855	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6856	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6857
6858	#ifdef VBOX_STRICT
6859	/* Validate the error-code-valid bit for hardware exceptions. */
6860	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6861	{
6862	switch (uVector)
6863	{
6864	case X86_XCPT_PF:
6865	case X86_XCPT_DF:
6866	case X86_XCPT_TS:
6867	case X86_XCPT_NP:
6868	case X86_XCPT_SS:
6869	case X86_XCPT_GP:
6870	case X86_XCPT_AC:
6871	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6872	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6873	/* fallthru */
6874	default:
6875	break;
6876	}
6877	}
6878	#endif
6879
6880	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6881	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6882	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6883
6884	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6885
6886	/* We require CR0 to check if the guest is in real-mode. */
6887	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6888	AssertRCReturn(rc, rc);
6889
6890	/*
6891	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6892	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6893	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6894	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6895	*/
6896	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6897	{
6898	PVM pVM = pVCpu->CTX_SUFF(pVM);
6899	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6900	{
6901	Assert(PDMVmmDevHeapIsEnabled(pVM));
6902	Assert(pVM->hm.s.vmx.pRealModeTSS);
6903
6904	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6905	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6906	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6907	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6908	AssertRCReturn(rc, rc);
6909	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6910
6911	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6912	const size_t cbIdtEntry = sizeof(X86IDTR16);
6913	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6914	{
6915	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6916	if (uVector == X86_XCPT_DF)
6917	return VINF_EM_RESET;
6918	else if (uVector == X86_XCPT_GP)
6919	{
6920	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6921	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6922	}
6923
6924	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6925	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6926	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6927	}
6928
6929	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6930	uint16_t uGuestIp = pMixedCtx->ip;
6931	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6932	{
6933	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6934	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6935	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6936	}
6937	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6938	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6939
6940	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6941	X86IDTR16 IdtEntry;
6942	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6943	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6944	AssertRCReturn(rc, rc);
6945
6946	/* Construct the stack frame for the interrupt/exception handler. */
6947	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6948	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6949	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6950	AssertRCReturn(rc, rc);
6951
6952	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6953	if (rc == VINF_SUCCESS)
6954	{
6955	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6956	pMixedCtx->rip = IdtEntry.offSel;
6957	pMixedCtx->cs.Sel = IdtEntry.uSel;
6958	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6959	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6960	&& uVector == X86_XCPT_PF)
6961	{
6962	pMixedCtx->cr2 = GCPtrFaultAddress;
6963	}
6964
6965	/* If any other guest-state bits are changed here, make sure to update
6966	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6967	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
6968	\| HM_CHANGED_GUEST_RIP
6969	\| HM_CHANGED_GUEST_RFLAGS
6970	\| HM_CHANGED_GUEST_RSP);
6971
6972	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6973	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6974	{
6975	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6976	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6977	Log4(("Clearing inhibition due to STI.\n"));
6978	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6979	}
6980	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6981
6982	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
6983	it, if we are returning to ring-3 before executing guest code. */
6984	pVCpu->hm.s.Event.fPending = false;
6985	}
6986	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6987	return rc;
6988	}
6989	else
6990	{
6991	/*
6992	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6993	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6994	*/
6995	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6996	}
6997	}
6998
6999	/* Validate. */
7000	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7001	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
7002	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7003
7004	/* Inject. */
7005	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
7006	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
7007	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7008	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7009
7010	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7011	&& uVector == X86_XCPT_PF)
7012	{
7013	pMixedCtx->cr2 = GCPtrFaultAddress;
7014	}
7015
7016	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7017	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7018
7019	AssertRCReturn(rc, rc);
7020	return rc;
7021	}
7022
7023
7024	/**
7025	* Clears the interrupt-window exiting control in the VMCS and if necessary
7026	* clears the current event in the VMCS as well.
7027	*
7028	* @returns VBox status code.
7029	* @param pVCpu Pointer to the VMCPU.
7030	*
7031	* @remarks Use this function only to clear events that have not yet been
7032	* delivered to the guest but are injected in the VMCS!
7033	* @remarks No-long-jump zone!!!
7034	*/
7035	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7036	{
7037	int rc;
7038	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7039
7040	/* Clear interrupt-window exiting control. */
7041	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7042	{
7043	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7044	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7045	AssertRC(rc);
7046	}
7047
7048	if (!pVCpu->hm.s.Event.fPending)
7049	return;
7050
7051	#ifdef VBOX_STRICT
7052	uint32_t u32EntryInfo;
7053	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7054	AssertRC(rc);
7055	Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
7056	#endif
7057
7058	/* Clear the entry-interruption field (including the valid bit). */
7059	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7060	AssertRC(rc);
7061
7062	/* Clear the pending debug exception field. */
7063	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7064	AssertRC(rc);
7065	}
7066
7067
7068	/**
7069	* Enters the VT-x session.
7070	*
7071	* @returns VBox status code.
7072	* @param pVM Pointer to the VM.
7073	* @param pVCpu Pointer to the VMCPU.
7074	* @param pCpu Pointer to the CPU info struct.
7075	*/
7076	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7077	{
7078	AssertPtr(pVM);
7079	AssertPtr(pVCpu);
7080	Assert(pVM->hm.s.vmx.fSupported);
7081	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7082	NOREF(pCpu);
7083
7084	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7085	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7086
7087	#ifdef VBOX_STRICT
7088	/* Make sure we're in VMX root mode. */
7089	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7090	if (!(u32HostCR4 & X86_CR4_VMXE))
7091	{
7092	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7093	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7094	}
7095	#endif
7096
7097	/*
7098	* Load the VCPU's VMCS as the current (and active) one.
7099	*/
7100	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7101	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7102	if (RT_FAILURE(rc))
7103	return rc;
7104
7105	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7106	pVCpu->hm.s.fLeaveDone = false;
7107	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7108
7109	return VINF_SUCCESS;
7110	}
7111
7112
7113	/**
7114	* The thread-context callback (only on platforms which support it).
7115	*
7116	* @param enmEvent The thread-context event.
7117	* @param pVCpu Pointer to the VMCPU.
7118	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7119	* @thread EMT.
7120	*/
7121	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7122	{
7123	switch (enmEvent)
7124	{
7125	case RTTHREADCTXEVENT_PREEMPTING:
7126	{
7127	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7128	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7129	VMCPU_ASSERT_EMT(pVCpu);
7130
7131	PVM pVM = pVCpu->CTX_SUFF(pVM);
7132	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7133
7134	/* No longjmps (logger flushes, locks) in this fragile context. */
7135	VMMRZCallRing3Disable(pVCpu);
7136	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7137
7138	/*
7139	* Restore host-state (FPU, debug etc.)
7140	*/
7141	if (!pVCpu->hm.s.fLeaveDone)
7142	{
7143	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7144	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7145	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7146	pVCpu->hm.s.fLeaveDone = true;
7147	}
7148
7149	/* Leave HM context, takes care of local init (term). */
7150	int rc = HMR0LeaveCpu(pVCpu);
7151	AssertRC(rc); NOREF(rc);
7152
7153	/* Restore longjmp state. */
7154	VMMRZCallRing3Enable(pVCpu);
7155	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7156	break;
7157	}
7158
7159	case RTTHREADCTXEVENT_RESUMED:
7160	{
7161	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7162	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7163	VMCPU_ASSERT_EMT(pVCpu);
7164
7165	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7166	VMMRZCallRing3Disable(pVCpu);
7167	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7168
7169	/* Initialize the bare minimum state required for HM. This takes care of
7170	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7171	int rc = HMR0EnterCpu(pVCpu);
7172	AssertRC(rc);
7173	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7174
7175	/* Load the active VMCS as the current one. */
7176	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7177	{
7178	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7179	AssertRC(rc); NOREF(rc);
7180	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7181	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7182	}
7183	pVCpu->hm.s.fLeaveDone = false;
7184
7185	/* Restore longjmp state. */
7186	VMMRZCallRing3Enable(pVCpu);
7187	break;
7188	}
7189
7190	default:
7191	break;
7192	}
7193	}
7194
7195
7196	/**
7197	* Saves the host state in the VMCS host-state.
7198	* Sets up the VM-exit MSR-load area.
7199	*
7200	* The CPU state will be loaded from these fields on every successful VM-exit.
7201	*
7202	* @returns VBox status code.
7203	* @param pVM Pointer to the VM.
7204	* @param pVCpu Pointer to the VMCPU.
7205	*
7206	* @remarks No-long-jump zone!!!
7207	*/
7208	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7209	{
7210	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7211
7212	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7213	return VINF_SUCCESS;
7214
7215	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7216	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7217
7218	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7219	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7220
7221	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7222	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7223
7224	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7225	return rc;
7226	}
7227
7228
7229	/**
7230	* Saves the host state in the VMCS host-state.
7231	*
7232	* @returns VBox status code.
7233	* @param pVM Pointer to the VM.
7234	* @param pVCpu Pointer to the VMCPU.
7235	*
7236	* @remarks No-long-jump zone!!!
7237	*/
7238	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7239	{
7240	AssertPtr(pVM);
7241	AssertPtr(pVCpu);
7242
7243	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7244
7245	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7246	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7247	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7248	return hmR0VmxSaveHostState(pVM, pVCpu);
7249	}
7250
7251
7252	/**
7253	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7254	* loaded from these fields on every successful VM-entry.
7255	*
7256	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7257	* Sets up the VM-entry controls.
7258	* Sets up the appropriate VMX non-root function to execute guest code based on
7259	* the guest CPU mode.
7260	*
7261	* @returns VBox status code.
7262	* @param pVM Pointer to the VM.
7263	* @param pVCpu Pointer to the VMCPU.
7264	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7265	* out-of-sync. Make sure to update the required fields
7266	* before using them.
7267	*
7268	* @remarks No-long-jump zone!!!
7269	*/
7270	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7271	{
7272	AssertPtr(pVM);
7273	AssertPtr(pVCpu);
7274	AssertPtr(pMixedCtx);
7275	HMVMX_ASSERT_PREEMPT_SAFE();
7276
7277	#ifdef LOG_ENABLED
7278	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7279	* probably not initialized yet? Anyway this will do for now.
7280	*
7281	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7282	* interface and disable ring-3 calls when thread-context hooks are not
7283	* available. */
7284	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7285	VMMR0LogFlushDisable(pVCpu);
7286	#endif
7287
7288	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7289
7290	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7291
7292	/* Determine real-on-v86 mode. */
7293	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7294	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7295	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7296	{
7297	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7298	}
7299
7300	/*
7301	* Load the guest-state into the VMCS.
7302	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7303	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7304	*/
7305	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7306	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7307
7308	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7309	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7311
7312	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7313	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7314	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7315
7316	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7317	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7318
7319	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7320	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7321
7322	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7323	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7324	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7325
7326	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7327	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7328
7329	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7330	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7331
7332	/*
7333	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7334	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7335	*/
7336	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7338
7339	/* Clear any unused and reserved bits. */
7340	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7341
7342	#ifdef LOG_ENABLED
7343	/* Only reenable log-flushing if the caller has it enabled. */
7344	if (!fCallerDisabledLogFlush)
7345	VMMR0LogFlushEnable(pVCpu);
7346	#endif
7347
7348	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7349	return rc;
7350	}
7351
7352
7353	/**
7354	* Loads the state shared between the host and guest into the VMCS.
7355	*
7356	* @param pVM Pointer to the VM.
7357	* @param pVCpu Pointer to the VMCPU.
7358	* @param pCtx Pointer to the guest-CPU context.
7359	*
7360	* @remarks No-long-jump zone!!!
7361	*/
7362	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7363	{
7364	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7365	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7366
7367	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7368	{
7369	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7370	AssertRC(rc);
7371	}
7372
7373	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7374	{
7375	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7376	AssertRC(rc);
7377
7378	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7379	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7380	{
7381	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7382	AssertRC(rc);
7383	}
7384	}
7385
7386	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7387	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7388	}
7389
7390
7391	/**
7392	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7393	*
7394	* @param pVM Pointer to the VM.
7395	* @param pVCpu Pointer to the VMCPU.
7396	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7397	* out-of-sync. Make sure to update the required fields
7398	* before using them.
7399	*/
7400	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7401	{
7402	HMVMX_ASSERT_PREEMPT_SAFE();
7403
7404	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7405	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7406	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7407	#endif
7408
7409	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7410	{
7411	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7412	AssertRC(rc);
7413	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7414	}
7415	else if (VMCPU_HMCF_VALUE(pVCpu))
7416	{
7417	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7418	AssertRC(rc);
7419	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7420	}
7421
7422	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7423	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7424	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7425	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7426
7427	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7428	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7429	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7430	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7431	#endif
7432	}
7433
7434
7435	/**
7436	* Does the preparations before executing guest code in VT-x.
7437	*
7438	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7439	* recompiler. We must be cautious what we do here regarding committing
7440	* guest-state information into the VMCS assuming we assuredly execute the
7441	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7442	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7443	* so that the recompiler can (and should) use them when it resumes guest
7444	* execution. Otherwise such operations must be done when we can no longer
7445	* exit to ring-3.
7446	*
7447	* @returns Strict VBox status code.
7448	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7449	* have been disabled.
7450	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7451	* double-fault into the guest.
7452	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7453	*
7454	* @param pVM Pointer to the VM.
7455	* @param pVCpu Pointer to the VMCPU.
7456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7457	* out-of-sync. Make sure to update the required fields
7458	* before using them.
7459	* @param pVmxTransient Pointer to the VMX transient structure.
7460	*
7461	* @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7462	* interrupts will be disabled.
7463	*/
7464	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7465	{
7466	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7467
7468	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7469	PGMRZDynMapFlushAutoSet(pVCpu);
7470	#endif
7471
7472	/* Check force flag actions that might require us to go back to ring-3. */
7473	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7474	if (rc != VINF_SUCCESS)
7475	return rc;
7476
7477	#ifndef IEM_VERIFICATION_MODE_FULL
7478	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7479	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7480	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7481	{
7482	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7483	RTGCPHYS GCPhysApicBase;
7484	GCPhysApicBase = pMixedCtx->msrApicBase;
7485	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7486
7487	/* Unalias any existing mapping. */
7488	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7489	AssertRCReturn(rc, rc);
7490
7491	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7492	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7493	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7494	AssertRCReturn(rc, rc);
7495
7496	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7497	}
7498	#endif /* !IEM_VERIFICATION_MODE_FULL */
7499
7500	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7501	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7502
7503	/*
7504	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7505	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7506	*/
7507	if (TRPMHasTrap(pVCpu))
7508	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7509	else if (!pVCpu->hm.s.Event.fPending)
7510	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7511
7512	/*
7513	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7514	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7515	*/
7516	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7517	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7518	{
7519	Assert(rc == VINF_EM_RESET);
7520	return rc;
7521	}
7522
7523	/*
7524	* No longjmps to ring-3 from this point on!!!
7525	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7526	* This also disables flushing of the R0-logger instance (if any).
7527	*/
7528	VMMRZCallRing3Disable(pVCpu);
7529
7530	/*
7531	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7532	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7533	*
7534	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7535	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7536	*
7537	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7538	* executing guest code.
7539	*/
7540	pVmxTransient->uEflags = ASMIntDisableFlags();
7541	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7542	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7543	{
7544	hmR0VmxClearEventVmcs(pVCpu);
7545	ASMSetFlags(pVmxTransient->uEflags);
7546	VMMRZCallRing3Enable(pVCpu);
7547	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7548	return VINF_EM_RAW_TO_R3;
7549	}
7550	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7551	{
7552	hmR0VmxClearEventVmcs(pVCpu);
7553	ASMSetFlags(pVmxTransient->uEflags);
7554	VMMRZCallRing3Enable(pVCpu);
7555	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7556	return VINF_EM_RAW_INTERRUPT;
7557	}
7558
7559	/* We've injected any pending events. This is really the point of no return (to ring-3). */
7560	pVCpu->hm.s.Event.fPending = false;
7561
7562	return VINF_SUCCESS;
7563	}
7564
7565
7566	/**
7567	* Prepares to run guest code in VT-x and we've committed to doing so. This
7568	* means there is no backing out to ring-3 or anywhere else at this
7569	* point.
7570	*
7571	* @param pVM Pointer to the VM.
7572	* @param pVCpu Pointer to the VMCPU.
7573	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7574	* out-of-sync. Make sure to update the required fields
7575	* before using them.
7576	* @param pVmxTransient Pointer to the VMX transient structure.
7577	*
7578	* @remarks Called with preemption disabled.
7579	* @remarks No-long-jump zone!!!
7580	*/
7581	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7582	{
7583	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7584	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7585	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7586
7587	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7588	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7589
7590	/*
7591	* If we are injecting events to a real-on-v86 mode guest, we may have to update
7592	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7593	* Reload only the necessary state, the assertion will catch if other parts of the code
7594	* change.
7595	*/
7596	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7597	{
7598	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7599	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7600	}
7601
7602	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7603	if (!CPUMIsGuestFPUStateActive(pVCpu))
7604	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7605	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7606	#endif
7607
7608	/*
7609	* Load the host state bits as we may've been preempted (only happens when
7610	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7611	*/
7612	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7613	{
7614	/* This ASSUMES that pfnStartVM has been set up already. */
7615	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7616	AssertRC(rc);
7617	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7618	}
7619	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7620
7621	/*
7622	* Load the state shared between host and guest (FPU, debug).
7623	*/
7624	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7625	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7626	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7627
7628	/* Store status of the shared guest-host state at the time of VM-entry. */
7629	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7630	if (CPUMIsGuestInLongModeEx(pMixedCtx))
7631	{
7632	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7633	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7634	}
7635	else
7636	#endif
7637	{
7638	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7639	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7640	}
7641	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7642
7643	/*
7644	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7645	*/
7646	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7647	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7648
7649	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7650	RTCPUID idCurrentCpu = pCpu->idCpu;
7651	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7652	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
7653	{
7654	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7655	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7656	}
7657
7658	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7659	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7660	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7661	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7662
7663	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7664
7665	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7666	to start executing. */
7667
7668	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7669	/*
7670	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7671	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7672	*/
7673	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7674	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7675	{
7676	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7677	uint64_t u64HostTscAux = 0;
7678	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7679	AssertRC(rc2);
7680	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7681	}
7682	#endif
7683	}
7684
7685
7686	/**
7687	* Performs some essential restoration of state after running guest code in
7688	* VT-x.
7689	*
7690	* @param pVM Pointer to the VM.
7691	* @param pVCpu Pointer to the VMCPU.
7692	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7693	* out-of-sync. Make sure to update the required fields
7694	* before using them.
7695	* @param pVmxTransient Pointer to the VMX transient structure.
7696	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7697	*
7698	* @remarks Called with interrupts disabled, and returns with interrups enabled!
7699	*
7700	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7701	* unconditionally when it is safe to do so.
7702	*/
7703	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7704	{
7705	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7706
7707	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7708	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7709	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7710	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7711	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7712
7713	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7714	{
7715	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7716	/* Restore host's TSC_AUX. */
7717	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7718	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7719	#endif
7720	/** @todo Find a way to fix hardcoding a guestimate. */
7721	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7722	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7723	}
7724
7725	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7726	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7727	Assert(!(ASMGetFlags() & X86_EFL_IF));
7728	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7729
7730	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7731	if (CPUMIsGuestFPUStateActive(pVCpu))
7732	{
7733	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7734	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7735	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7736	}
7737	#endif
7738
7739	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7740	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7741	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7742	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7743
7744	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7745	uint32_t uExitReason;
7746	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7747	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7748	AssertRC(rc);
7749	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7750	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7751
7752	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7753	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7754	{
7755	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7756	pVmxTransient->fVMEntryFailed));
7757	return;
7758	}
7759
7760	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7761	{
7762	/* Update the guest interruptibility-state from the VMCS. */
7763	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7764	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7765	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7766	AssertRC(rc);
7767	#endif
7768	/*
7769	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7770	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7771	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7772	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7773	*/
7774	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7775	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7776	{
7777	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7778	AssertRC(rc);
7779	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
7780	}
7781	}
7782	}
7783
7784
7785
7786	/**
7787	* Runs the guest code using VT-x the normal way.
7788	*
7789	* @returns VBox status code.
7790	* @param pVM Pointer to the VM.
7791	* @param pVCpu Pointer to the VMCPU.
7792	* @param pCtx Pointer to the guest-CPU context.
7793	*
7794	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
7795	* @remarks Called with preemption disabled.
7796	*/
7797	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7798	{
7799	VMXTRANSIENT VmxTransient;
7800	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7801	int rc = VERR_INTERNAL_ERROR_5;
7802	uint32_t cLoops = 0;
7803
7804	for (;; cLoops++)
7805	{
7806	Assert(!HMR0SuspendPending());
7807	HMVMX_ASSERT_CPU_SAFE();
7808
7809	/* Preparatory work for running guest code, this may force us to return
7810	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7811	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7812	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7813	if (rc != VINF_SUCCESS)
7814	break;
7815
7816	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7817	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7818	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7819
7820	/* Restore any residual host-state and save any bits shared between host
7821	and guest into the guest-CPU state. Re-enables interrupts! */
7822	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7823
7824	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7825	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7826	{
7827	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7828	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7829	return rc;
7830	}
7831
7832	/* Handle the VM-exit. */
7833	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7834	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7835	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7836	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7837	HMVMX_START_EXIT_DISPATCH_PROF();
7838	#ifdef HMVMX_USE_FUNCTION_TABLE
7839	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7840	#else
7841	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7842	#endif
7843	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7844	if (rc != VINF_SUCCESS)
7845	break;
7846	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7847	{
7848	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7849	rc = VINF_EM_RAW_INTERRUPT;
7850	break;
7851	}
7852	}
7853
7854	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7855	return rc;
7856	}
7857
7858
7859	/**
7860	* Single steps guest code using VT-x.
7861	*
7862	* @returns VBox status code.
7863	* @param pVM Pointer to the VM.
7864	* @param pVCpu Pointer to the VMCPU.
7865	* @param pCtx Pointer to the guest-CPU context.
7866	*
7867	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7868	* @remarks Called with preemption disabled.
7869	*/
7870	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7871	{
7872	VMXTRANSIENT VmxTransient;
7873	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7874	int rc = VERR_INTERNAL_ERROR_5;
7875	uint32_t cLoops = 0;
7876	uint16_t uCsStart = pCtx->cs.Sel;
7877	uint64_t uRipStart = pCtx->rip;
7878
7879	for (;; cLoops++)
7880	{
7881	Assert(!HMR0SuspendPending());
7882	HMVMX_ASSERT_CPU_SAFE();
7883
7884	/* Preparatory work for running guest code, this may force us to return
7885	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7886	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7887	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7888	if (rc != VINF_SUCCESS)
7889	break;
7890
7891	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7892	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7893	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7894
7895	/* Restore any residual host-state and save any bits shared between host
7896	and guest into the guest-CPU state. Re-enables interrupts! */
7897	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7898
7899	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7900	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7901	{
7902	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7903	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7904	return rc;
7905	}
7906
7907	/* Handle the VM-exit. */
7908	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7909	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7910	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7911	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7912	HMVMX_START_EXIT_DISPATCH_PROF();
7913	#ifdef HMVMX_USE_FUNCTION_TABLE
7914	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7915	#else
7916	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7917	#endif
7918	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7919	if (rc != VINF_SUCCESS)
7920	break;
7921	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7922	{
7923	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7924	rc = VINF_EM_RAW_INTERRUPT;
7925	break;
7926	}
7927
7928	/*
7929	* Did the RIP change, if so, consider it a single step.
7930	* Otherwise, make sure one of the TFs gets set.
7931	*/
7932	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7933	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7934	AssertRCReturn(rc2, rc2);
7935	if ( pCtx->rip != uRipStart
7936	\|\| pCtx->cs.Sel != uCsStart)
7937	{
7938	rc = VINF_EM_DBG_STEPPED;
7939	break;
7940	}
7941	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7942	}
7943
7944	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7945	return rc;
7946	}
7947
7948
7949	/**
7950	* Runs the guest code using VT-x.
7951	*
7952	* @returns VBox status code.
7953	* @param pVM Pointer to the VM.
7954	* @param pVCpu Pointer to the VMCPU.
7955	* @param pCtx Pointer to the guest-CPU context.
7956	*
7957	* @remarks Called with preemption disabled.
7958	*/
7959	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7960	{
7961	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7962	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7963	HMVMX_ASSERT_PREEMPT_SAFE();
7964
7965	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7966
7967	int rc;
7968	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7969	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7970	else
7971	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7972
7973	if (rc == VERR_EM_INTERPRETER)
7974	rc = VINF_EM_RAW_EMULATE_INSTR;
7975	else if (rc == VINF_EM_RESET)
7976	rc = VINF_EM_TRIPLE_FAULT;
7977
7978	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7979	if (RT_FAILURE(rc2))
7980	{
7981	pVCpu->hm.s.u32HMError = rc;
7982	rc = rc2;
7983	}
7984	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7985	return rc;
7986	}
7987
7988
7989	#ifndef HMVMX_USE_FUNCTION_TABLE
7990	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7991	{
7992	int rc;
7993	switch (rcReason)
7994	{
7995	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7996	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7997	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7998	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7999	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8000	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8001	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8002	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8003	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8004	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8005	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8006	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8007	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8008	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8009	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8010	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8011	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8012	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8013	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8014	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8015	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8016	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8017	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8018	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8019	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8020	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8021	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8022	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8023	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8024	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8025	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8026	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8027	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8028
8029	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8030	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8031	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8032	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8033	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8034	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8035	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8036	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8037	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8038
8039	case VMX_EXIT_VMCALL:
8040	case VMX_EXIT_VMCLEAR:
8041	case VMX_EXIT_VMLAUNCH:
8042	case VMX_EXIT_VMPTRLD:
8043	case VMX_EXIT_VMPTRST:
8044	case VMX_EXIT_VMREAD:
8045	case VMX_EXIT_VMRESUME:
8046	case VMX_EXIT_VMWRITE:
8047	case VMX_EXIT_VMXOFF:
8048	case VMX_EXIT_VMXON:
8049	case VMX_EXIT_INVEPT:
8050	case VMX_EXIT_INVVPID:
8051	case VMX_EXIT_VMFUNC:
8052	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8053	break;
8054	default:
8055	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8056	break;
8057	}
8058	return rc;
8059	}
8060	#endif
8061
8062	#ifdef DEBUG
8063	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8064	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8065	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8066
8067	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8068	do \
8069	{ \
8070	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8071	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8072	} while (0)
8073
8074	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8075	do { \
8076	AssertPtr(pVCpu); \
8077	AssertPtr(pMixedCtx); \
8078	AssertPtr(pVmxTransient); \
8079	Assert(pVmxTransient->fVMEntryFailed == false); \
8080	Assert(ASMIntAreEnabled()); \
8081	HMVMX_ASSERT_PREEMPT_SAFE(); \
8082	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8083	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8084	HMVMX_ASSERT_PREEMPT_SAFE(); \
8085	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8086	HMVMX_ASSERT_PREEMPT_CPUID(); \
8087	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8088	} while (0)
8089
8090	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8091	do { \
8092	Log4Func(("\n")); \
8093	} while(0)
8094	#else /* Release builds */
8095	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8096	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8097	#endif
8098
8099
8100	/**
8101	* Advances the guest RIP after reading it from the VMCS.
8102	*
8103	* @returns VBox status code.
8104	* @param pVCpu Pointer to the VMCPU.
8105	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8106	* out-of-sync. Make sure to update the required fields
8107	* before using them.
8108	* @param pVmxTransient Pointer to the VMX transient structure.
8109	*
8110	* @remarks No-long-jump zone!!!
8111	*/
8112	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8113	{
8114	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8115	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8116	AssertRCReturn(rc, rc);
8117
8118	pMixedCtx->rip += pVmxTransient->cbInstr;
8119	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8120	return rc;
8121	}
8122
8123
8124	/**
8125	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8126	* and update error record fields accordingly.
8127	*
8128	* @return VMX_IGS_* return codes.
8129	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8130	* wrong with the guest state.
8131	*
8132	* @param pVM Pointer to the VM.
8133	* @param pVCpu Pointer to the VMCPU.
8134	* @param pCtx Pointer to the guest-CPU state.
8135	*/
8136	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8137	{
8138	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8139	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8140	uError = (err); \
8141	break; \
8142	} else do {} while (0)
8143	/* Duplicate of IEM_IS_CANONICAL(). */
8144	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8145
8146	int rc;
8147	uint32_t uError = VMX_IGS_ERROR;
8148	uint32_t u32Val;
8149	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8150
8151	do
8152	{
8153	/*
8154	* CR0.
8155	*/
8156	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8157	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8158	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8159	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8160	if (fUnrestrictedGuest)
8161	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8162
8163	uint32_t u32GuestCR0;
8164	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8165	AssertRCBreak(rc);
8166	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8167	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8168	if ( !fUnrestrictedGuest
8169	&& (u32GuestCR0 & X86_CR0_PG)
8170	&& !(u32GuestCR0 & X86_CR0_PE))
8171	{
8172	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8173	}
8174
8175	/*
8176	* CR4.
8177	*/
8178	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8179	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8180
8181	uint32_t u32GuestCR4;
8182	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8183	AssertRCBreak(rc);
8184	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8185	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8186
8187	/*
8188	* IA32_DEBUGCTL MSR.
8189	*/
8190	uint64_t u64Val;
8191	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8192	AssertRCBreak(rc);
8193	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8194	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8195	{
8196	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8197	}
8198	uint64_t u64DebugCtlMsr = u64Val;
8199
8200	#ifdef VBOX_STRICT
8201	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8202	AssertRCBreak(rc);
8203	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8204	#endif
8205	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8206
8207	/*
8208	* RIP and RFLAGS.
8209	*/
8210	uint32_t u32Eflags;
8211	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8212	if (HMVMX_IS_64BIT_HOST_MODE())
8213	{
8214	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8215	AssertRCBreak(rc);
8216	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8217	if ( !fLongModeGuest
8218	\|\| !pCtx->cs.Attr.n.u1Long)
8219	{
8220	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8221	}
8222	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8223	* must be identical if the "IA32e mode guest" VM-entry control is 1
8224	* and CS.L is 1. No check applies if the CPU supports 64
8225	* linear-address bits. */
8226
8227	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8228	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8229	AssertRCBreak(rc);
8230	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8231	VMX_IGS_RFLAGS_RESERVED);
8232	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8233	u32Eflags = u64Val;
8234	}
8235	else
8236	#endif
8237	{
8238	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8239	AssertRCBreak(rc);
8240	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8241	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8242	}
8243
8244	if ( fLongModeGuest
8245	\|\| ( fUnrestrictedGuest
8246	&& !(u32GuestCR0 & X86_CR0_PE)))
8247	{
8248	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8249	}
8250
8251	uint32_t u32EntryInfo;
8252	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8253	AssertRCBreak(rc);
8254	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8255	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8256	{
8257	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8258	}
8259
8260	/*
8261	* 64-bit checks.
8262	*/
8263	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8264	if (HMVMX_IS_64BIT_HOST_MODE())
8265	{
8266	if ( fLongModeGuest
8267	&& !fUnrestrictedGuest)
8268	{
8269	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8270	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8271	}
8272
8273	if ( !fLongModeGuest
8274	&& (u32GuestCR4 & X86_CR4_PCIDE))
8275	{
8276	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8277	}
8278
8279	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8280	* 51:32 beyond the processor's physical-address width are 0. */
8281
8282	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8283	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8284	{
8285	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8286	}
8287
8288	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8289	AssertRCBreak(rc);
8290	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8291
8292	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8293	AssertRCBreak(rc);
8294	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8295	}
8296	#endif
8297
8298	/*
8299	* PERF_GLOBAL MSR.
8300	*/
8301	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8302	{
8303	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8304	AssertRCBreak(rc);
8305	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8306	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8307	}
8308
8309	/*
8310	* PAT MSR.
8311	*/
8312	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8313	{
8314	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8315	AssertRCBreak(rc);
8316	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8317	for (unsigned i = 0; i < 8; i++)
8318	{
8319	uint8_t u8Val = (u64Val & 0x7);
8320	if ( u8Val != 0 /* UC */
8321	\|\| u8Val != 1 /* WC */
8322	\|\| u8Val != 4 /* WT */
8323	\|\| u8Val != 5 /* WP */
8324	\|\| u8Val != 6 /* WB */
8325	\|\| u8Val != 7 /* UC- */)
8326	{
8327	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8328	}
8329	u64Val >>= 3;
8330	}
8331	}
8332
8333	/*
8334	* EFER MSR.
8335	*/
8336	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8337	{
8338	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8339	AssertRCBreak(rc);
8340	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8341	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8342	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8343	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8344	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8345	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8346	}
8347
8348	/*
8349	* Segment registers.
8350	*/
8351	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8352	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8353	if (!(u32Eflags & X86_EFL_VM))
8354	{
8355	/* CS */
8356	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8357	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8358	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8359	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8360	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8361	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8362	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8363	/* CS cannot be loaded with NULL in protected mode. */
8364	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8365	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8366	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8367	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8368	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8369	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8370	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8371	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8372	else
8373	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8374
8375	/* SS */
8376	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8377	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8378	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8379	if ( !(pCtx->cr0 & X86_CR0_PE)
8380	\|\| pCtx->cs.Attr.n.u4Type == 3)
8381	{
8382	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8383	}
8384	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8385	{
8386	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8387	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8388	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8389	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8390	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8391	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8392	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8393	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8394	}
8395
8396	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8397	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8398	{
8399	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8400	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8401	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8402	\|\| pCtx->ds.Attr.n.u4Type > 11
8403	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8404	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8405	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8406	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8407	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8408	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8409	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8410	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8411	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8412	}
8413	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8414	{
8415	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8416	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8417	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8418	\|\| pCtx->es.Attr.n.u4Type > 11
8419	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8420	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8421	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8422	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8423	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8424	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8425	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8426	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8427	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8428	}
8429	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8430	{
8431	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8432	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8433	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8434	\|\| pCtx->fs.Attr.n.u4Type > 11
8435	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8436	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8437	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8438	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8439	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8440	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8441	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8442	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8443	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8444	}
8445	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8446	{
8447	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8448	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8449	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8450	\|\| pCtx->gs.Attr.n.u4Type > 11
8451	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8452	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8453	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8454	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8455	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8456	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8457	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8458	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8459	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8460	}
8461	/* 64-bit capable CPUs. */
8462	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8463	if (HMVMX_IS_64BIT_HOST_MODE())
8464	{
8465	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8466	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8467	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8468	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8469	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8470	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8471	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8472	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8473	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8474	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8475	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8476	}
8477	#endif
8478	}
8479	else
8480	{
8481	/* V86 mode checks. */
8482	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8483	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8484	{
8485	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8486	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8487	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8488	}
8489	else
8490	{
8491	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8492	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8493	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8494	}
8495
8496	/* CS */
8497	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8498	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8499	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8500	/* SS */
8501	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8502	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8503	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8504	/* DS */
8505	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8506	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8507	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8508	/* ES */
8509	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8510	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8511	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8512	/* FS */
8513	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8514	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8515	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8516	/* GS */
8517	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8518	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8519	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8520	/* 64-bit capable CPUs. */
8521	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8522	if (HMVMX_IS_64BIT_HOST_MODE())
8523	{
8524	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8525	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8526	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8527	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8528	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8529	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8530	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8531	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8532	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8533	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8534	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8535	}
8536	#endif
8537	}
8538
8539	/*
8540	* TR.
8541	*/
8542	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8543	/* 64-bit capable CPUs. */
8544	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8545	if (HMVMX_IS_64BIT_HOST_MODE())
8546	{
8547	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8548	}
8549	#endif
8550	if (fLongModeGuest)
8551	{
8552	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8553	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8554	}
8555	else
8556	{
8557	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8558	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8559	VMX_IGS_TR_ATTR_TYPE_INVALID);
8560	}
8561	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8562	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8563	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8564	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8565	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8566	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8567	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8568	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8569
8570	/*
8571	* GDTR and IDTR.
8572	*/
8573	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8574	if (HMVMX_IS_64BIT_HOST_MODE())
8575	{
8576	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8577	AssertRCBreak(rc);
8578	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8579
8580	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8581	AssertRCBreak(rc);
8582	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8583	}
8584	#endif
8585
8586	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8587	AssertRCBreak(rc);
8588	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8589
8590	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8591	AssertRCBreak(rc);
8592	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8593
8594	/*
8595	* Guest Non-Register State.
8596	*/
8597	/* Activity State. */
8598	uint32_t u32ActivityState;
8599	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8600	AssertRCBreak(rc);
8601	HMVMX_CHECK_BREAK( !u32ActivityState
8602	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8603	VMX_IGS_ACTIVITY_STATE_INVALID);
8604	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8605	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8606	uint32_t u32IntrState;
8607	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8608	AssertRCBreak(rc);
8609	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8610	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8611	{
8612	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8613	}
8614
8615	/** @todo Activity state and injecting interrupts. Left as a todo since we
8616	* currently don't use activity states but ACTIVE. */
8617
8618	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8619	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8620
8621	/* Guest interruptibility-state. */
8622	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8623	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8624	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8625	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8626	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8627	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8628	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8629	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8630	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8631	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8632	{
8633	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8634	{
8635	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8636	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8637	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8638	}
8639	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8640	{
8641	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8642	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8643	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8644	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8645	}
8646	}
8647	/** @todo Assumes the processor is not in SMM. */
8648	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8649	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8650	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8651	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8652	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8653	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8654	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8655	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8656	{
8657	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8658	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8659	}
8660
8661	/* Pending debug exceptions. */
8662	if (HMVMX_IS_64BIT_HOST_MODE())
8663	{
8664	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8665	AssertRCBreak(rc);
8666	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8667	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8668	u32Val = u64Val; /* For pending debug exceptions checks below. */
8669	}
8670	else
8671	{
8672	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8673	AssertRCBreak(rc);
8674	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8675	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8676	}
8677
8678	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8679	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8680	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8681	{
8682	if ( (u32Eflags & X86_EFL_TF)
8683	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8684	{
8685	/* Bit 14 is PendingDebug.BS. */
8686	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8687	}
8688	if ( !(u32Eflags & X86_EFL_TF)
8689	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8690	{
8691	/* Bit 14 is PendingDebug.BS. */
8692	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8693	}
8694	}
8695
8696	/* VMCS link pointer. */
8697	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8698	AssertRCBreak(rc);
8699	if (u64Val != UINT64_C(0xffffffffffffffff))
8700	{
8701	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8702	/** @todo Bits beyond the processor's physical-address width MBZ. */
8703	/** @todo 32-bit located in memory referenced by value of this field (as a
8704	* physical address) must contain the processor's VMCS revision ID. */
8705	/** @todo SMM checks. */
8706	}
8707
8708	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8709
8710	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8711	if (uError == VMX_IGS_ERROR)
8712	uError = VMX_IGS_REASON_NOT_FOUND;
8713	} while (0);
8714
8715	pVCpu->hm.s.u32HMError = uError;
8716	return uError;
8717
8718	#undef HMVMX_ERROR_BREAK
8719	#undef HMVMX_CHECK_BREAK
8720	#undef HMVMX_IS_CANONICAL
8721	}
8722
8723	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8724	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8725	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8726
8727	/** @name VM-exit handlers.
8728	* @{
8729	*/
8730
8731	/**
8732	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8733	*/
8734	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8735	{
8736	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8738	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8739	#if HC_ARCH_BITS == 64
8740	Assert(ASMIntAreEnabled());
8741	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8742	return VINF_SUCCESS;
8743	#endif
8744	return VINF_EM_RAW_INTERRUPT;
8745	}
8746
8747
8748	/**
8749	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8750	*/
8751	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8752	{
8753	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8754	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8755
8756	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8757	AssertRCReturn(rc, rc);
8758
8759	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8760	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8761	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8762	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8763
8764	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8765	{
8766	/*
8767	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8768	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8769	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8770	*
8771	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8772	*/
8773	VMXDispatchHostNmi();
8774	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8775	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8776	return VINF_SUCCESS;
8777	}
8778
8779	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8780	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8781	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8782	{
8783	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8784	return VINF_SUCCESS;
8785	}
8786	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8787	{
8788	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8789	return rc;
8790	}
8791
8792	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8793	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8794	switch (uIntrType)
8795	{
8796	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8797	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8798	/* no break */
8799	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8800	{
8801	switch (uVector)
8802	{
8803	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8804	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8805	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8806	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8807	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8808	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8809	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8810	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8811	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8812	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8813	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8814	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8815	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8816	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8817	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8818	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8819	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8820	#endif
8821	default:
8822	{
8823	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8824	AssertRCReturn(rc, rc);
8825
8826	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8827	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8828	{
8829	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8830	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8831	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8832
8833	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8834	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8835	AssertRCReturn(rc, rc);
8836	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8837	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8838	0 /* GCPtrFaultAddress */);
8839	AssertRCReturn(rc, rc);
8840	}
8841	else
8842	{
8843	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8844	pVCpu->hm.s.u32HMError = uVector;
8845	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8846	}
8847	break;
8848	}
8849	}
8850	break;
8851	}
8852
8853	default:
8854	{
8855	pVCpu->hm.s.u32HMError = uExitIntrInfo;
8856	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8857	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8858	break;
8859	}
8860	}
8861	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8862	return rc;
8863	}
8864
8865
8866	/**
8867	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8868	*/
8869	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8870	{
8871	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8872
8873	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8874	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8875	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8876	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8877	AssertRCReturn(rc, rc);
8878
8879	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8880	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8881	return VINF_SUCCESS;
8882	}
8883
8884
8885	/**
8886	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8887	*/
8888	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8889	{
8890	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8891	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8892	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8893	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8894	}
8895
8896
8897	/**
8898	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8899	*/
8900	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8901	{
8902	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8903	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8904	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8905	}
8906
8907
8908	/**
8909	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8910	*/
8911	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8912	{
8913	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8914	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8915	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8916	}
8917
8918
8919	/**
8920	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8921	*/
8922	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8923	{
8924	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8925	PVM pVM = pVCpu->CTX_SUFF(pVM);
8926	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8927	if (RT_LIKELY(rc == VINF_SUCCESS))
8928	{
8929	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8930	Assert(pVmxTransient->cbInstr == 2);
8931	}
8932	else
8933	{
8934	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8935	rc = VERR_EM_INTERPRETER;
8936	}
8937	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8938	return rc;
8939	}
8940
8941
8942	/**
8943	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8944	*/
8945	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8946	{
8947	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8948	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8949	AssertRCReturn(rc, rc);
8950
8951	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8952	return VINF_EM_RAW_EMULATE_INSTR;
8953
8954	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8955	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8956	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8957	}
8958
8959
8960	/**
8961	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8962	*/
8963	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8964	{
8965	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8966	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8967	AssertRCReturn(rc, rc);
8968
8969	PVM pVM = pVCpu->CTX_SUFF(pVM);
8970	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8971	if (RT_LIKELY(rc == VINF_SUCCESS))
8972	{
8973	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8974	Assert(pVmxTransient->cbInstr == 2);
8975	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8976	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8977	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8978	}
8979	else
8980	{
8981	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8982	rc = VERR_EM_INTERPRETER;
8983	}
8984	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8985	return rc;
8986	}
8987
8988
8989	/**
8990	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8991	*/
8992	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8993	{
8994	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8995	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8996	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8997	AssertRCReturn(rc, rc);
8998
8999	PVM pVM = pVCpu->CTX_SUFF(pVM);
9000	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9001	if (RT_LIKELY(rc == VINF_SUCCESS))
9002	{
9003	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9004	Assert(pVmxTransient->cbInstr == 3);
9005	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9006	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9007	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9008	}
9009	else
9010	{
9011	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9012	rc = VERR_EM_INTERPRETER;
9013	}
9014	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9015	return rc;
9016	}
9017
9018
9019	/**
9020	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9021	*/
9022	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9023	{
9024	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9025	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9026	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9027	AssertRCReturn(rc, rc);
9028
9029	PVM pVM = pVCpu->CTX_SUFF(pVM);
9030	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9031	if (RT_LIKELY(rc == VINF_SUCCESS))
9032	{
9033	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9034	Assert(pVmxTransient->cbInstr == 2);
9035	}
9036	else
9037	{
9038	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9039	rc = VERR_EM_INTERPRETER;
9040	}
9041	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9042	return rc;
9043	}
9044
9045
9046	/**
9047	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9048	*/
9049	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9050	{
9051	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9052	PVM pVM = pVCpu->CTX_SUFF(pVM);
9053	Assert(!pVM->hm.s.fNestedPaging);
9054
9055	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9056	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9057	AssertRCReturn(rc, rc);
9058
9059	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9060	rc = VBOXSTRICTRC_VAL(rc2);
9061	if (RT_LIKELY(rc == VINF_SUCCESS))
9062	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9063	else
9064	{
9065	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9066	pVmxTransient->uExitQualification, rc));
9067	}
9068	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9069	return rc;
9070	}
9071
9072
9073	/**
9074	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9075	*/
9076	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9077	{
9078	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9079	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9080	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9081	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9082	AssertRCReturn(rc, rc);
9083
9084	PVM pVM = pVCpu->CTX_SUFF(pVM);
9085	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9086	if (RT_LIKELY(rc == VINF_SUCCESS))
9087	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9088	else
9089	{
9090	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9091	rc = VERR_EM_INTERPRETER;
9092	}
9093	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9094	return rc;
9095	}
9096
9097
9098	/**
9099	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9100	*/
9101	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9102	{
9103	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9104	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9105	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9106	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9107	AssertRCReturn(rc, rc);
9108
9109	PVM pVM = pVCpu->CTX_SUFF(pVM);
9110	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9111	rc = VBOXSTRICTRC_VAL(rc2);
9112	if (RT_LIKELY( rc == VINF_SUCCESS
9113	\|\| rc == VINF_EM_HALT))
9114	{
9115	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9116	AssertRCReturn(rc3, rc3);
9117
9118	if ( rc == VINF_EM_HALT
9119	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9120	{
9121	rc = VINF_SUCCESS;
9122	}
9123	}
9124	else
9125	{
9126	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9127	rc = VERR_EM_INTERPRETER;
9128	}
9129	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9130	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9131	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9132	return rc;
9133	}
9134
9135
9136	/**
9137	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9138	*/
9139	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9140	{
9141	/*
9142	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9143	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9144	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9145	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9146	*/
9147	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9148	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
9149	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9150	}
9151
9152
9153	/**
9154	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9155	*/
9156	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9157	{
9158	/*
9159	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9160	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9161	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9162	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9163	*/
9164	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9165	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
9166	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9167	}
9168
9169
9170	/**
9171	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9172	*/
9173	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9174	{
9175	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9176	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9177	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
9178	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9179	}
9180
9181
9182	/**
9183	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9184	*/
9185	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9186	{
9187	/*
9188	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9189	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9190	* See Intel spec. 25.3 "Other Causes of VM-exits".
9191	*/
9192	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9193	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
9194	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9195	}
9196
9197
9198	/**
9199	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9200	* VM-exit.
9201	*/
9202	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9203	{
9204	/*
9205	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9206	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9207	*
9208	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9209	* See Intel spec. "23.8 Restrictions on VMX operation".
9210	*/
9211	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9212	return VINF_SUCCESS;
9213	}
9214
9215
9216	/**
9217	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9218	* VM-exit.
9219	*/
9220	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9221	{
9222	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9223	return VINF_EM_RESET;
9224	}
9225
9226
9227	/**
9228	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9229	*/
9230	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9231	{
9232	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9233	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9234	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9235	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9236	AssertRCReturn(rc, rc);
9237
9238	pMixedCtx->rip++;
9239	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9240	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9241	rc = VINF_SUCCESS;
9242	else
9243	rc = VINF_EM_HALT;
9244
9245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9246	return rc;
9247	}
9248
9249
9250	/**
9251	* VM-exit handler for instructions that result in a #UD exception delivered to
9252	* the guest.
9253	*/
9254	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9255	{
9256	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9257	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9258	return VINF_SUCCESS;
9259	}
9260
9261
9262	/**
9263	* VM-exit handler for expiry of the VMX preemption timer.
9264	*/
9265	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9266	{
9267	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9268
9269	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9270	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9271
9272	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9273	PVM pVM = pVCpu->CTX_SUFF(pVM);
9274	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9275	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9276	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9277	}
9278
9279
9280	/**
9281	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9282	*/
9283	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9284	{
9285	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9286
9287	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9288	/** @todo check if XSETBV is supported by the recompiler. */
9289	return VERR_EM_INTERPRETER;
9290	}
9291
9292
9293	/**
9294	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9295	*/
9296	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9297	{
9298	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9299
9300	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9301	/** @todo implement EMInterpretInvpcid() */
9302	return VERR_EM_INTERPRETER;
9303	}
9304
9305
9306	/**
9307	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9308	* Error VM-exit.
9309	*/
9310	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9311	{
9312	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9313	AssertRCReturn(rc, rc);
9314
9315	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9316	NOREF(uInvalidReason);
9317
9318	#ifdef VBOX_STRICT
9319	uint32_t uIntrState;
9320	HMVMXHCUINTREG uHCReg;
9321	uint64_t u64Val;
9322	uint32_t u32Val;
9323
9324	rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9325	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9326	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9327	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9328	AssertRCReturn(rc, rc);
9329
9330	Log4(("uInvalidReason %u\n", uInvalidReason));
9331	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9332	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9333	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9334	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9335
9336	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9337	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9338	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9339	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9340	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9341	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9342	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9343	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9344	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9345	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9346	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9347	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9348	#endif
9349
9350	PVM pVM = pVCpu->CTX_SUFF(pVM);
9351	HMDumpRegs(pVM, pVCpu, pMixedCtx);
9352
9353	return VERR_VMX_INVALID_GUEST_STATE;
9354	}
9355
9356
9357	/**
9358	* VM-exit handler for VM-entry failure due to an MSR-load
9359	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9360	*/
9361	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9362	{
9363	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9364	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9365	}
9366
9367
9368	/**
9369	* VM-exit handler for VM-entry failure due to a machine-check event
9370	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9371	*/
9372	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9373	{
9374	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9375	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9376	}
9377
9378
9379	/**
9380	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9381	* theory.
9382	*/
9383	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9384	{
9385	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9386	return VERR_VMX_UNDEFINED_EXIT_CODE;
9387	}
9388
9389
9390	/**
9391	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9392	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9393	* Conditional VM-exit.
9394	*/
9395	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9396	{
9397	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9398
9399	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9400	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9401	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9402	return VERR_EM_INTERPRETER;
9403	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9404	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9405	}
9406
9407
9408	/**
9409	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9410	*/
9411	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9412	{
9413	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9414
9415	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9417	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9418	return VERR_EM_INTERPRETER;
9419	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9420	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9421	}
9422
9423
9424	/**
9425	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9426	*/
9427	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9428	{
9429	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9430
9431	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9432	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9433	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9434	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9435	AssertRCReturn(rc, rc);
9436	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9437
9438	PVM pVM = pVCpu->CTX_SUFF(pVM);
9439	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9440	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9441	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9442	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9443
9444	if (RT_LIKELY(rc == VINF_SUCCESS))
9445	{
9446	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9447	Assert(pVmxTransient->cbInstr == 2);
9448	}
9449	return rc;
9450	}
9451
9452
9453	/**
9454	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9455	*/
9456	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9457	{
9458	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9459	PVM pVM = pVCpu->CTX_SUFF(pVM);
9460	int rc = VINF_SUCCESS;
9461
9462	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9463	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9464	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9465	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9466	AssertRCReturn(rc, rc);
9467	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9468
9469	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9470	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9471	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9472
9473	if (RT_LIKELY(rc == VINF_SUCCESS))
9474	{
9475	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9476
9477	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9478	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9479	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9480	{
9481	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9482	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9483	EMInterpretWrmsr() changes it. */
9484	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9485	}
9486	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9487	{
9488	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9489	AssertRCReturn(rc, rc);
9490	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9491	}
9492	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9493	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9494
9495	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9496	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9497	{
9498	switch (pMixedCtx->ecx)
9499	{
9500	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9501	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9502	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9503	case MSR_K8_FS_BASE: /* no break */
9504	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9505	case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9506	}
9507	}
9508	#ifdef VBOX_STRICT
9509	else
9510	{
9511	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9512	switch (pMixedCtx->ecx)
9513	{
9514	case MSR_IA32_SYSENTER_CS:
9515	case MSR_IA32_SYSENTER_EIP:
9516	case MSR_IA32_SYSENTER_ESP:
9517	case MSR_K8_FS_BASE:
9518	case MSR_K8_GS_BASE:
9519	{
9520	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9521	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9522	}
9523
9524	case MSR_K8_LSTAR:
9525	case MSR_K6_STAR:
9526	case MSR_K8_SF_MASK:
9527	case MSR_K8_TSC_AUX:
9528	case MSR_K8_KERNEL_GS_BASE:
9529	{
9530	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9531	pMixedCtx->ecx));
9532	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9533	}
9534	}
9535	}
9536	#endif /* VBOX_STRICT */
9537	}
9538	return rc;
9539	}
9540
9541
9542	/**
9543	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9544	*/
9545	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9546	{
9547	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9548
9549	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9550	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9551	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9552	return VERR_EM_INTERPRETER;
9553	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9554	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9555	}
9556
9557
9558	/**
9559	* VM-exit handler for when the TPR value is lowered below the specified
9560	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9561	*/
9562	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9563	{
9564	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9565	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9566
9567	/*
9568	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9569	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9570	* resume guest execution.
9571	*/
9572	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9573	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9574	return VINF_SUCCESS;
9575	}
9576
9577
9578	/**
9579	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9580	* VM-exit.
9581	*
9582	* @retval VINF_SUCCESS when guest execution can continue.
9583	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9584	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9585	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9586	* recompiler.
9587	*/
9588	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9589	{
9590	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9591	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9592	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9593	AssertRCReturn(rc, rc);
9594
9595	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9596	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9597	PVM pVM = pVCpu->CTX_SUFF(pVM);
9598	switch (uAccessType)
9599	{
9600	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9601	{
9602	#if 0
9603	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9604	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9605	#else
9606	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9607	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9608	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9609	#endif
9610	AssertRCReturn(rc, rc);
9611
9612	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9613	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9614	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9615	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9616
9617	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9618	{
9619	case 0: /* CR0 */
9620	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9621	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9622	break;
9623	case 2: /* C2 **/
9624	/* Nothing to do here, CR2 it's not part of the VMCS. */
9625	break;
9626	case 3: /* CR3 */
9627	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9628	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9629	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9630	break;
9631	case 4: /* CR4 */
9632	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9633	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9634	break;
9635	case 8: /* CR8 */
9636	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9637	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9638	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9639	break;
9640	default:
9641	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9642	break;
9643	}
9644
9645	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9646	break;
9647	}
9648
9649	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9650	{
9651	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9652	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9653	AssertRCReturn(rc, rc);
9654	Assert( !pVM->hm.s.fNestedPaging
9655	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9656	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9657
9658	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9659	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9660	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9661
9662	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9663	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9664	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9665	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9667	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9668	break;
9669	}
9670
9671	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9672	{
9673	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9674	AssertRCReturn(rc, rc);
9675	rc = EMInterpretCLTS(pVM, pVCpu);
9676	AssertRCReturn(rc, rc);
9677	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9679	Log4(("CRX CLTS write rc=%d\n", rc));
9680	break;
9681	}
9682
9683	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9684	{
9685	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9686	AssertRCReturn(rc, rc);
9687	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9688	if (RT_LIKELY(rc == VINF_SUCCESS))
9689	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9690	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9691	Log4(("CRX LMSW write rc=%d\n", rc));
9692	break;
9693	}
9694
9695	default:
9696	{
9697	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9698	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9699	}
9700	}
9701
9702	/* Validate possible error codes. */
9703	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9704	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9705	if (RT_SUCCESS(rc))
9706	{
9707	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9708	AssertRCReturn(rc2, rc2);
9709	}
9710
9711	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9712	return rc;
9713	}
9714
9715
9716	/**
9717	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9718	* VM-exit.
9719	*/
9720	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9721	{
9722	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9723	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9724
9725	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9726	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9727	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9728	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9729	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9730	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9731	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9732	AssertRCReturn(rc2, rc2);
9733
9734	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9735	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9736	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9737	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9738	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9739	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9740	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9741
9742	/* I/O operation lookup arrays. */
9743	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9744	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9745
9746	VBOXSTRICTRC rcStrict;
9747	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9748	const uint32_t cbInstr = pVmxTransient->cbInstr;
9749	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9750	PVM pVM = pVCpu->CTX_SUFF(pVM);
9751	if (fIOString)
9752	{
9753	/*
9754	* INS/OUTS - I/O String instruction.
9755	*
9756	* Use instruction-information if available, otherwise fall back on
9757	* interpreting the instruction.
9758	*/
9759	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9760	#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9761	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9762	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9763	{
9764	rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9765	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9766	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9767	AssertRCReturn(rc2, rc2);
9768	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9769	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9770	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9771	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9772	if (fIOWrite)
9773	{
9774	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9775	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9776	//if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9777	// hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9778	// pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9779	}
9780	else
9781	{
9782	AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9783	("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9784	VERR_HMVMX_IPE_4);
9785	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9786	//if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9787	// hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9788	}
9789	}
9790	else
9791	{
9792	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9793	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9794	AssertRCReturn(rc2, rc2);
9795	rcStrict = IEMExecOne(pVCpu);
9796	}
9797	/** @todo IEM needs to be setting these flags somehow. */
9798	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9799	fUpdateRipAlready = true;
9800	#else
9801	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9802	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9803	if (RT_SUCCESS(rcStrict))
9804	{
9805	if (fIOWrite)
9806	{
9807	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9808	(DISCPUMODE)pDis->uAddrMode, cbValue);
9809	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9810	}
9811	else
9812	{
9813	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9814	(DISCPUMODE)pDis->uAddrMode, cbValue);
9815	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9816	}
9817	}
9818	else
9819	{
9820	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9821	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9822	}
9823	#endif
9824	}
9825	else
9826	{
9827	/*
9828	* IN/OUT - I/O instruction.
9829	*/
9830	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9831	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9832	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9833	if (fIOWrite)
9834	{
9835	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9836	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9837	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9838	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9839	}
9840	else
9841	{
9842	uint32_t u32Result = 0;
9843	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9844	if (IOM_SUCCESS(rcStrict))
9845	{
9846	/* Save result of I/O IN instr. in AL/AX/EAX. */
9847	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9848	}
9849	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9850	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9851	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9852	}
9853	}
9854
9855	if (IOM_SUCCESS(rcStrict))
9856	{
9857	if (!fUpdateRipAlready)
9858	{
9859	pMixedCtx->rip += cbInstr;
9860	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9861	}
9862
9863	/*
9864	* If any I/O breakpoints are armed, we need to check if one triggered
9865	* and take appropriate action.
9866	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9867	*/
9868	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9869	AssertRCReturn(rc2, rc2);
9870
9871	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9872	* execution engines about whether hyper BPs and such are pending. */
9873	uint32_t const uDr7 = pMixedCtx->dr[7];
9874	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9875	&& X86_DR7_ANY_RW_IO(uDr7)
9876	&& (pMixedCtx->cr4 & X86_CR4_DE))
9877	\|\| DBGFBpIsHwIoArmed(pVM)))
9878	{
9879	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9880
9881	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
9882	VMMRZCallRing3Disable(pVCpu);
9883	HM_DISABLE_PREEMPT_IF_NEEDED();
9884
9885	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9886
9887	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9888	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9889	{
9890	/* Raise #DB. */
9891	if (fIsGuestDbgActive)
9892	ASMSetDR6(pMixedCtx->dr[6]);
9893	if (pMixedCtx->dr[7] != uDr7)
9894	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9895
9896	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9897	}
9898	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9899	else if ( rcStrict2 != VINF_SUCCESS
9900	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9901	rcStrict = rcStrict2;
9902
9903	HM_RESTORE_PREEMPT_IF_NEEDED();
9904	VMMRZCallRing3Enable(pVCpu);
9905	}
9906	}
9907
9908	#ifdef DEBUG
9909	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9910	Assert(!fIOWrite);
9911	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9912	Assert(fIOWrite);
9913	else
9914	{
9915	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9916	* statuses, that the VMM device and some others may return. See
9917	* IOM_SUCCESS() for guidance. */
9918	AssertMsg( RT_FAILURE(rcStrict)
9919	\|\| rcStrict == VINF_SUCCESS
9920	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9921	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9922	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9923	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9924	}
9925	#endif
9926
9927	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9928	return VBOXSTRICTRC_TODO(rcStrict);
9929	}
9930
9931
9932	/**
9933	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9934	* VM-exit.
9935	*/
9936	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9937	{
9938	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9939
9940	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9941	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9942	AssertRCReturn(rc, rc);
9943	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9944	{
9945	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9946	AssertRCReturn(rc, rc);
9947	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9948	{
9949	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9950
9951	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9952	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9953	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9954	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9955	{
9956	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9957	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9958
9959	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9960	Assert(!pVCpu->hm.s.Event.fPending);
9961	pVCpu->hm.s.Event.fPending = true;
9962	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9963	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9964	AssertRCReturn(rc, rc);
9965	if (fErrorCodeValid)
9966	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9967	else
9968	pVCpu->hm.s.Event.u32ErrCode = 0;
9969	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9970	&& uVector == X86_XCPT_PF)
9971	{
9972	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9973	}
9974
9975	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9976	}
9977	}
9978	}
9979
9980	/** @todo Emulate task switch someday, currently just going back to ring-3 for
9981	* emulation. */
9982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9983	return VERR_EM_INTERPRETER;
9984	}
9985
9986
9987	/**
9988	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9989	*/
9990	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9991	{
9992	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9993	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9994	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9995	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9996	AssertRCReturn(rc, rc);
9997	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9998	return VINF_EM_DBG_STEPPED;
9999	}
10000
10001
10002	/**
10003	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10004	*/
10005	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10006	{
10007	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10008
10009	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10010	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10011	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10012	return VINF_SUCCESS;
10013	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10014	return rc;
10015
10016	#if 0
10017	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10018	* just sync the whole thing. */
10019	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10020	#else
10021	/* Aggressive state sync. for now. */
10022	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10023	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10024	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10025	#endif
10026	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10027	AssertRCReturn(rc, rc);
10028
10029	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10030	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10031	switch (uAccessType)
10032	{
10033	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10034	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10035	{
10036	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10037	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10038	{
10039	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10040	}
10041
10042	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10043	GCPhys &= PAGE_BASE_GC_MASK;
10044	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10045	PVM pVM = pVCpu->CTX_SUFF(pVM);
10046	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10047	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10048
10049	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10050	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10051	CPUMCTX2CORE(pMixedCtx), GCPhys);
10052	rc = VBOXSTRICTRC_VAL(rc2);
10053	Log4(("ApicAccess rc=%d\n", rc));
10054	if ( rc == VINF_SUCCESS
10055	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10056	\|\| rc == VERR_PAGE_NOT_PRESENT)
10057	{
10058	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10059	\| HM_CHANGED_GUEST_RSP
10060	\| HM_CHANGED_GUEST_RFLAGS
10061	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10062	rc = VINF_SUCCESS;
10063	}
10064	break;
10065	}
10066
10067	default:
10068	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10069	rc = VINF_EM_RAW_EMULATE_INSTR;
10070	break;
10071	}
10072
10073	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10074	return rc;
10075	}
10076
10077
10078	/**
10079	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10080	* VM-exit.
10081	*/
10082	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10083	{
10084	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10085
10086	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10087	if (pVmxTransient->fWasGuestDebugStateActive)
10088	{
10089	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10090	return VERR_VMX_UNEXPECTED_EXIT_CODE;
10091	}
10092
10093	int rc = VERR_INTERNAL_ERROR_5;
10094	if ( !DBGFIsStepping(pVCpu)
10095	&& !pVCpu->hm.s.fSingleInstruction
10096	&& !pVmxTransient->fWasHyperDebugStateActive)
10097	{
10098	/* Don't intercept MOV DRx and #DB any more. */
10099	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10100	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10101	AssertRCReturn(rc, rc);
10102
10103	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10104	{
10105	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10106	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10107	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10108	AssertRCReturn(rc, rc);
10109	#endif
10110	}
10111
10112	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10113	VMMRZCallRing3Disable(pVCpu);
10114	HM_DISABLE_PREEMPT_IF_NEEDED();
10115
10116	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10117	PVM pVM = pVCpu->CTX_SUFF(pVM);
10118	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10119	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10120
10121	HM_RESTORE_PREEMPT_IF_NEEDED();
10122	VMMRZCallRing3Enable(pVCpu);
10123
10124	#ifdef VBOX_WITH_STATISTICS
10125	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10126	AssertRCReturn(rc, rc);
10127	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10128	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10129	else
10130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10131	#endif
10132	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10133	return VINF_SUCCESS;
10134	}
10135
10136	/*
10137	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10138	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10139	*/
10140	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10141	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10142	AssertRCReturn(rc, rc);
10143	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10144
10145	PVM pVM = pVCpu->CTX_SUFF(pVM);
10146	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10147	{
10148	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10149	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10150	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10151	if (RT_SUCCESS(rc))
10152	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10153	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10154	}
10155	else
10156	{
10157	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10158	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10159	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10160	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10161	}
10162
10163	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10164	if (RT_SUCCESS(rc))
10165	{
10166	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10167	AssertRCReturn(rc2, rc2);
10168	}
10169	return rc;
10170	}
10171
10172
10173	/**
10174	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10175	* Conditional VM-exit.
10176	*/
10177	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10178	{
10179	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10180	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10181
10182	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10183	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10184	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10185	return VINF_SUCCESS;
10186	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10187	return rc;
10188
10189	RTGCPHYS GCPhys = 0;
10190	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10191
10192	#if 0
10193	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10194	#else
10195	/* Aggressive state sync. for now. */
10196	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10197	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10198	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10199	#endif
10200	AssertRCReturn(rc, rc);
10201
10202	/*
10203	* If we succeed, resume guest execution.
10204	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10205	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10206	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10207	* weird case. See @bugref{6043}.
10208	*/
10209	PVM pVM = pVCpu->CTX_SUFF(pVM);
10210	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10211	rc = VBOXSTRICTRC_VAL(rc2);
10212	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10213	if ( rc == VINF_SUCCESS
10214	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10215	\|\| rc == VERR_PAGE_NOT_PRESENT)
10216	{
10217	/* Successfully handled MMIO operation. */
10218	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10219	\| HM_CHANGED_GUEST_RSP
10220	\| HM_CHANGED_GUEST_RFLAGS
10221	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10222	rc = VINF_SUCCESS;
10223	}
10224	return rc;
10225	}
10226
10227
10228	/**
10229	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10230	* VM-exit.
10231	*/
10232	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10233	{
10234	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10235	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10236
10237	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10238	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10239	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10240	return VINF_SUCCESS;
10241	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10242	return rc;
10243
10244	RTGCPHYS GCPhys = 0;
10245	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10246	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10247	#if 0
10248	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10249	#else
10250	/* Aggressive state sync. for now. */
10251	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10252	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10253	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10254	#endif
10255	AssertRCReturn(rc, rc);
10256
10257	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10258	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10259
10260	RTGCUINT uErrorCode = 0;
10261	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10262	uErrorCode \|= X86_TRAP_PF_ID;
10263	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10264	uErrorCode \|= X86_TRAP_PF_RW;
10265	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10266	uErrorCode \|= X86_TRAP_PF_P;
10267
10268	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10269
10270	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10271	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10272
10273	/* Handle the pagefault trap for the nested shadow table. */
10274	PVM pVM = pVCpu->CTX_SUFF(pVM);
10275	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10276	TRPMResetTrap(pVCpu);
10277
10278	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10279	if ( rc == VINF_SUCCESS
10280	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10281	\|\| rc == VERR_PAGE_NOT_PRESENT)
10282	{
10283	/* Successfully synced our nested page tables. */
10284	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10285	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10286	\| HM_CHANGED_GUEST_RSP
10287	\| HM_CHANGED_GUEST_RFLAGS);
10288	return VINF_SUCCESS;
10289	}
10290
10291	Log4(("EPT return to ring-3 rc=%d\n"));
10292	return rc;
10293	}
10294
10295	/** @} */
10296
10297	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10298	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10299	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10300
10301	/** @name VM-exit exception handlers.
10302	* @{
10303	*/
10304
10305	/**
10306	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10307	*/
10308	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10309	{
10310	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10312
10313	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10314	AssertRCReturn(rc, rc);
10315
10316	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10317	{
10318	/* Old-style FPU error reporting needs some extra work. */
10319	/** @todo don't fall back to the recompiler, but do it manually. */
10320	return VERR_EM_INTERPRETER;
10321	}
10322
10323	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10324	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10325	return rc;
10326	}
10327
10328
10329	/**
10330	* VM-exit exception handler for #BP (Breakpoint exception).
10331	*/
10332	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10333	{
10334	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10335	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10336
10337	/** @todo Try optimize this by not saving the entire guest state unless
10338	* really needed. */
10339	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10340	AssertRCReturn(rc, rc);
10341
10342	PVM pVM = pVCpu->CTX_SUFF(pVM);
10343	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10344	if (rc == VINF_EM_RAW_GUEST_TRAP)
10345	{
10346	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10347	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10348	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10349	AssertRCReturn(rc, rc);
10350
10351	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10352	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10353	}
10354
10355	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10356	return rc;
10357	}
10358
10359
10360	/**
10361	* VM-exit exception handler for #DB (Debug exception).
10362	*/
10363	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10364	{
10365	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10366	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10367	Log6(("XcptDB\n"));
10368
10369	/*
10370	* Get the DR6-like values from the exit qualification and pass it to DBGF
10371	* for processing.
10372	*/
10373	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10374	AssertRCReturn(rc, rc);
10375
10376	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10377	uint64_t uDR6 = X86_DR6_INIT_VAL;
10378	uDR6 \|= ( pVmxTransient->uExitQualification
10379	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10380
10381	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10382	if (rc == VINF_EM_RAW_GUEST_TRAP)
10383	{
10384	/*
10385	* The exception was for the guest. Update DR6, DR7.GD and
10386	* IA32_DEBUGCTL.LBR before forwarding it.
10387	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10388	*/
10389	VMMRZCallRing3Disable(pVCpu);
10390	HM_DISABLE_PREEMPT_IF_NEEDED();
10391
10392	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10393	pMixedCtx->dr[6] \|= uDR6;
10394	if (CPUMIsGuestDebugStateActive(pVCpu))
10395	ASMSetDR6(pMixedCtx->dr[6]);
10396
10397	HM_RESTORE_PREEMPT_IF_NEEDED();
10398	VMMRZCallRing3Enable(pVCpu);
10399
10400	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10401	AssertRCReturn(rc, rc);
10402
10403	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10404	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10405
10406	/* Paranoia. */
10407	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10408	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10409
10410	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10411	AssertRCReturn(rc, rc);
10412
10413	/*
10414	* Raise #DB in the guest.
10415	*/
10416	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10417	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10418	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10419	AssertRCReturn(rc, rc);
10420	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10421	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10422	return VINF_SUCCESS;
10423	}
10424
10425	/*
10426	* Not a guest trap, must be a hypervisor related debug event then.
10427	* Update DR6 in case someone is interested in it.
10428	*/
10429	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10430	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10431	CPUMSetHyperDR6(pVCpu, uDR6);
10432
10433	return rc;
10434	}
10435
10436
10437	/**
10438	* VM-exit exception handler for #NM (Device-not-available exception: floating
10439	* point exception).
10440	*/
10441	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10442	{
10443	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10444
10445	/* We require CR0 and EFER. EFER is always up-to-date. */
10446	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10447	AssertRCReturn(rc, rc);
10448
10449	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10450	VMMRZCallRing3Disable(pVCpu);
10451	HM_DISABLE_PREEMPT_IF_NEEDED();
10452
10453	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10454	if (pVmxTransient->fWasGuestFPUStateActive)
10455	{
10456	rc = VINF_EM_RAW_GUEST_TRAP;
10457	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10458	}
10459	else
10460	{
10461	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10462	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10463	#endif
10464	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10465	rc = CPUMR0LoadGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10466	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10467	}
10468
10469	HM_RESTORE_PREEMPT_IF_NEEDED();
10470	VMMRZCallRing3Enable(pVCpu);
10471
10472	if (rc == VINF_SUCCESS)
10473	{
10474	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10475	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10476	}
10477	else
10478	{
10479	/* Forward #NM to the guest. */
10480	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10481	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10482	AssertRCReturn(rc, rc);
10483	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10484	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
10485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10486	}
10487
10488	return VINF_SUCCESS;
10489	}
10490
10491
10492	/**
10493	* VM-exit exception handler for #GP (General-protection exception).
10494	*
10495	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10496	*/
10497	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10498	{
10499	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10500	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10501
10502	int rc = VERR_INTERNAL_ERROR_5;
10503	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10504	{
10505	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10506	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10507	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10508	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10509	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10510	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10511	AssertRCReturn(rc, rc);
10512	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10513	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10514	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10515	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10516	return rc;
10517	#else
10518	/* We don't intercept #GP. */
10519	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10520	return VERR_VMX_UNEXPECTED_EXCEPTION;
10521	#endif
10522	}
10523
10524	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10525	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10526
10527	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10528	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10529	AssertRCReturn(rc, rc);
10530
10531	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10532	uint32_t cbOp = 0;
10533	PVM pVM = pVCpu->CTX_SUFF(pVM);
10534	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10535	if (RT_SUCCESS(rc))
10536	{
10537	rc = VINF_SUCCESS;
10538	Assert(cbOp == pDis->cbInstr);
10539	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10540	switch (pDis->pCurInstr->uOpcode)
10541	{
10542	case OP_CLI:
10543	{
10544	pMixedCtx->eflags.Bits.u1IF = 0;
10545	pMixedCtx->rip += pDis->cbInstr;
10546	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10547	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10548	break;
10549	}
10550
10551	case OP_STI:
10552	{
10553	pMixedCtx->eflags.Bits.u1IF = 1;
10554	pMixedCtx->rip += pDis->cbInstr;
10555	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10556	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10557	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10558	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10559	break;
10560	}
10561
10562	case OP_HLT:
10563	{
10564	rc = VINF_EM_HALT;
10565	pMixedCtx->rip += pDis->cbInstr;
10566	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10568	break;
10569	}
10570
10571	case OP_POPF:
10572	{
10573	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10574	uint32_t cbParm = 0;
10575	uint32_t uMask = 0;
10576	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10577	{
10578	cbParm = 4;
10579	uMask = 0xffffffff;
10580	}
10581	else
10582	{
10583	cbParm = 2;
10584	uMask = 0xffff;
10585	}
10586
10587	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10588	RTGCPTR GCPtrStack = 0;
10589	X86EFLAGS Eflags;
10590	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10591	&GCPtrStack);
10592	if (RT_SUCCESS(rc))
10593	{
10594	Assert(sizeof(Eflags.u32) >= cbParm);
10595	Eflags.u32 = 0;
10596	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10597	}
10598	if (RT_FAILURE(rc))
10599	{
10600	rc = VERR_EM_INTERPRETER;
10601	break;
10602	}
10603	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10604	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10605	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10606	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10607	pMixedCtx->esp += cbParm;
10608	pMixedCtx->esp &= uMask;
10609	pMixedCtx->rip += pDis->cbInstr;
10610
10611	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10612	\| HM_CHANGED_GUEST_RSP
10613	\| HM_CHANGED_GUEST_RFLAGS);
10614	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10615	break;
10616	}
10617
10618	case OP_PUSHF:
10619	{
10620	uint32_t cbParm = 0;
10621	uint32_t uMask = 0;
10622	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10623	{
10624	cbParm = 4;
10625	uMask = 0xffffffff;
10626	}
10627	else
10628	{
10629	cbParm = 2;
10630	uMask = 0xffff;
10631	}
10632
10633	/* Get the stack pointer & push the contents of eflags onto the stack. */
10634	RTGCPTR GCPtrStack = 0;
10635	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10636	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10637	if (RT_FAILURE(rc))
10638	{
10639	rc = VERR_EM_INTERPRETER;
10640	break;
10641	}
10642	X86EFLAGS Eflags = pMixedCtx->eflags;
10643	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10644	Eflags.Bits.u1RF = 0;
10645	Eflags.Bits.u1VM = 0;
10646
10647	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10648	if (RT_FAILURE(rc))
10649	{
10650	rc = VERR_EM_INTERPRETER;
10651	break;
10652	}
10653	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10654	pMixedCtx->esp -= cbParm;
10655	pMixedCtx->esp &= uMask;
10656	pMixedCtx->rip += pDis->cbInstr;
10657	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
10658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10659	break;
10660	}
10661
10662	case OP_IRET:
10663	{
10664	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10665	* instruction reference. */
10666	RTGCPTR GCPtrStack = 0;
10667	uint32_t uMask = 0xffff;
10668	uint16_t aIretFrame[3];
10669	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10670	{
10671	rc = VERR_EM_INTERPRETER;
10672	break;
10673	}
10674	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10675	&GCPtrStack);
10676	if (RT_SUCCESS(rc))
10677	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10678	if (RT_FAILURE(rc))
10679	{
10680	rc = VERR_EM_INTERPRETER;
10681	break;
10682	}
10683	pMixedCtx->eip = 0;
10684	pMixedCtx->ip = aIretFrame[0];
10685	pMixedCtx->cs.Sel = aIretFrame[1];
10686	pMixedCtx->cs.ValidSel = aIretFrame[1];
10687	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10688	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10689	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10690	pMixedCtx->sp += sizeof(aIretFrame);
10691	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10692	\| HM_CHANGED_GUEST_SEGMENT_REGS
10693	\| HM_CHANGED_GUEST_RSP
10694	\| HM_CHANGED_GUEST_RFLAGS);
10695	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10696	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10697	break;
10698	}
10699
10700	case OP_INT:
10701	{
10702	uint16_t uVector = pDis->Param1.uValue & 0xff;
10703	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10704	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10705	break;
10706	}
10707
10708	case OP_INTO:
10709	{
10710	if (pMixedCtx->eflags.Bits.u1OF)
10711	{
10712	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10713	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10714	}
10715	break;
10716	}
10717
10718	default:
10719	{
10720	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10721	EMCODETYPE_SUPERVISOR);
10722	rc = VBOXSTRICTRC_VAL(rc2);
10723	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10724	Log4(("#GP rc=%Rrc\n", rc));
10725	break;
10726	}
10727	}
10728	}
10729	else
10730	rc = VERR_EM_INTERPRETER;
10731
10732	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10733	("#GP Unexpected rc=%Rrc\n", rc));
10734	return rc;
10735	}
10736
10737
10738	/**
10739	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10740	* the exception reported in the VMX transient structure back into the VM.
10741	*
10742	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
10743	* up-to-date.
10744	*/
10745	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10746	{
10747	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10748
10749	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10750	hmR0VmxCheckExitDueToEventDelivery(). */
10751	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10752	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10753	AssertRCReturn(rc, rc);
10754	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10755
10756	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10757	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10758	return VINF_SUCCESS;
10759	}
10760
10761
10762	/**
10763	* VM-exit exception handler for #PF (Page-fault exception).
10764	*/
10765	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10766	{
10767	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10768	PVM pVM = pVCpu->CTX_SUFF(pVM);
10769	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10770	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10771	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10772	AssertRCReturn(rc, rc);
10773
10774	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10775	if (pVM->hm.s.fNestedPaging)
10776	{
10777	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10778	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10779	{
10780	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10781	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10782	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10783	}
10784	else
10785	{
10786	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10787	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10788	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10789	}
10790	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10791	return rc;
10792	}
10793	#else
10794	Assert(!pVM->hm.s.fNestedPaging);
10795	#endif
10796
10797	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10798	AssertRCReturn(rc, rc);
10799
10800	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10801	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10802
10803	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10804	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10805	(RTGCPTR)pVmxTransient->uExitQualification);
10806
10807	Log4(("#PF: rc=%Rrc\n", rc));
10808	if (rc == VINF_SUCCESS)
10809	{
10810	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10811	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10812	* memory? We don't update the whole state here... */
10813	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10814	\| HM_CHANGED_GUEST_RSP
10815	\| HM_CHANGED_GUEST_RFLAGS
10816	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10817	TRPMResetTrap(pVCpu);
10818	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10819	return rc;
10820	}
10821	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10822	{
10823	if (!pVmxTransient->fVectoringPF)
10824	{
10825	/* It's a guest page fault and needs to be reflected to the guest. */
10826	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10827	TRPMResetTrap(pVCpu);
10828	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10829	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10830	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10831	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10832	}
10833	else
10834	{
10835	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10836	TRPMResetTrap(pVCpu);
10837	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10838	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10839	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10840	}
10841
10842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10843	return VINF_SUCCESS;
10844	}
10845
10846	TRPMResetTrap(pVCpu);
10847	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10848	return rc;
10849	}
10850
10851	/** @} */
10852

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48749

Download in other formats: