HMVMXR0.cpp@ 49074

Last change on this file since 49074 was 49026, checked in by vboxsync, 11 years ago
VMM: Some @thread doxygen corrections.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 452.9 KB

Line
1	/* $Id: HMVMXR0.cpp 49026 2013-10-10 12:01:59Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#define HMVMX_ALWAYS_SWAP_FPU_STATE
45	#endif
46
47
48	/*******************************************************************************
49	* Defined Constants And Macros *
50	*******************************************************************************/
51	#if defined(RT_ARCH_AMD64)
52	# define HMVMX_IS_64BIT_HOST_MODE() (true)
53	typedef RTHCUINTREG HMVMXHCUINTREG;
54	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
55	extern "C" uint32_t g_fVMXIs64bitHost;
56	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57	typedef uint64_t HMVMXHCUINTREG;
58	#else
59	# define HMVMX_IS_64BIT_HOST_MODE() (false)
60	typedef RTHCUINTREG HMVMXHCUINTREG;
61	#endif
62
63	/** Use the function table. */
64	#define HMVMX_USE_FUNCTION_TABLE
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
213	uint64_t u64LStarMsr;
214	/** The guest's TPR value used for TPR shadowing. */
215	uint8_t u8GuestTpr;
216	/** Alignment. */
217	uint8_t abAlignment0[7];
218
219	/** The basic VM-exit reason. */
220	uint16_t uExitReason;
221	/** Alignment. */
222	uint16_t u16Alignment0;
223	/** The VM-exit interruption error code. */
224	uint32_t uExitIntErrorCode;
225	/** The VM-exit exit qualification. */
226	uint64_t uExitQualification;
227
228	/** The VM-exit interruption-information field. */
229	uint32_t uExitIntInfo;
230	/** The VM-exit instruction-length field. */
231	uint32_t cbInstr;
232	/** The VM-exit instruction-information field. */
233	union
234	{
235	/** Plain unsigned int representation. */
236	uint32_t u;
237	/** INS and OUTS information. */
238	struct
239	{
240	uint32_t u6Reserved0 : 7;
241	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242	uint32_t u3AddrSize : 3;
243	uint32_t u5Reserved1 : 5;
244	/** The segment register (X86_SREG_XXX). */
245	uint32_t iSegReg : 3;
246	uint32_t uReserved2 : 14;
247	} StrIo;
248	} ExitInstrInfo;
249	/** Whether the VM-entry failed or not. */
250	bool fVMEntryFailed;
251	/** Alignment. */
252	uint8_t abAlignment1[3];
253
254	/** The VM-entry interruption-information field. */
255	uint32_t uEntryIntInfo;
256	/** The VM-entry exception error code field. */
257	uint32_t uEntryXcptErrorCode;
258	/** The VM-entry instruction length field. */
259	uint32_t cbEntryInstr;
260
261	/** IDT-vectoring information field. */
262	uint32_t uIdtVectoringInfo;
263	/** IDT-vectoring error code. */
264	uint32_t uIdtVectoringErrorCode;
265
266	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
267	uint32_t fVmcsFieldsRead;
268
269	/** Whether the guest FPU was active at the time of VM-exit. */
270	bool fWasGuestFPUStateActive;
271	/** Whether the guest debug state was active at the time of VM-exit. */
272	bool fWasGuestDebugStateActive;
273	/** Whether the hyper debug state was active at the time of VM-exit. */
274	bool fWasHyperDebugStateActive;
275	/** Whether TSC-offsetting should be setup before VM-entry. */
276	bool fUpdateTscOffsettingAndPreemptTimer;
277	/** Whether the VM-exit was caused by a page-fault during delivery of a
278	* contributory exception or a page-fault. */
279	bool fVectoringPF;
280	} VMXTRANSIENT;
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286	/** Pointer to VMX transient state. */
287	typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290	/**
291	* MSR-bitmap read permissions.
292	*/
293	typedef enum VMXMSREXITREAD
294	{
295	/** Reading this MSR causes a VM-exit. */
296	VMXMSREXIT_INTERCEPT_READ = 0xb,
297	/** Reading this MSR does not cause a VM-exit. */
298	VMXMSREXIT_PASSTHRU_READ
299	} VMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311
312	/**
313	* VMX VM-exit handler.
314	*
315	* @returns VBox status code.
316	* @param pVCpu Pointer to the VMCPU.
317	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
318	* out-of-sync. Make sure to update the required
319	* fields before using them.
320	* @param pVmxTransient Pointer to the VMX-transient structure.
321	*/
322	#ifndef HMVMX_USE_FUNCTION_TABLE
323	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
324	#else
325	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
326	/** Pointer to VM-exit handler. */
327	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
328	#endif
329
330
331	/*******************************************************************************
332	* Internal Functions *
333	*******************************************************************************/
334	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
335	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
336	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
337	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
338	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
339	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
340	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
341	#endif
342	#ifndef HMVMX_USE_FUNCTION_TABLE
343	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
344	# define HMVMX_EXIT_DECL static int
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
347	#endif
348
349	/** @name VM-exit handlers.
350	* @{
351	*/
352	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
353	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
354	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
355	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
356	static FNVMXEXITHANDLER hmR0VmxExitSipi;
357	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
358	static FNVMXEXITHANDLER hmR0VmxExitSmi;
359	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
360	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
361	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
362	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
363	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
364	static FNVMXEXITHANDLER hmR0VmxExitHlt;
365	static FNVMXEXITHANDLER hmR0VmxExitInvd;
366	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
367	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
368	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
369	static FNVMXEXITHANDLER hmR0VmxExitRsm;
370	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
371	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
372	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
373	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
374	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
375	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
376	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
377	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
378	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
379	static FNVMXEXITHANDLER hmR0VmxExitMwait;
380	static FNVMXEXITHANDLER hmR0VmxExitMtf;
381	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
382	static FNVMXEXITHANDLER hmR0VmxExitPause;
383	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
384	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
385	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
386	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
387	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
389	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
390	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
391	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
392	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
393	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
394	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
395	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
396	/** @} */
397
398	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
399	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
400	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
406
407	/*******************************************************************************
408	* Global Variables *
409	*******************************************************************************/
410	#ifdef HMVMX_USE_FUNCTION_TABLE
411
412	/**
413	* VMX_EXIT dispatch table.
414	*/
415	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
416	{
417	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
418	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
419	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
420	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
421	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
422	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
423	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
424	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
425	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
426	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
427	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
428	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
429	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
430	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
431	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
432	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
433	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
434	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
435	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
436	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
437	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
438	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
439	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
440	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
441	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
442	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
443	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
444	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
445	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
446	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
447	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
448	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
449	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
450	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
451	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
452	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
453	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
454	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
455	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
456	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
457	/* 40 UNDEFINED */ hmR0VmxExitPause,
458	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
459	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
460	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
461	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
462	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
463	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
464	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
465	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
466	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
467	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
468	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
469	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
470	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
471	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
472	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
473	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
474	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
475	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
476	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
477	};
478	#endif /* HMVMX_USE_FUNCTION_TABLE */
479
480	#ifdef VBOX_STRICT
481	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
482	{
483	/* 0 */ "(Not Used)",
484	/* 1 */ "VMCALL executed in VMX root operation.",
485	/* 2 */ "VMCLEAR with invalid physical address.",
486	/* 3 */ "VMCLEAR with VMXON pointer.",
487	/* 4 */ "VMLAUNCH with non-clear VMCS.",
488	/* 5 */ "VMRESUME with non-launched VMCS.",
489	/* 6 */ "VMRESUME after VMXOFF",
490	/* 7 */ "VM entry with invalid control fields.",
491	/* 8 */ "VM entry with invalid host state fields.",
492	/* 9 */ "VMPTRLD with invalid physical address.",
493	/* 10 */ "VMPTRLD with VMXON pointer.",
494	/* 11 */ "VMPTRLD with incorrect revision identifier.",
495	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
496	/* 13 */ "VMWRITE to read-only VMCS component.",
497	/* 14 */ "(Not Used)",
498	/* 15 */ "VMXON executed in VMX root operation.",
499	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
500	/* 17 */ "VM entry with non-launched executing VMCS.",
501	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
502	/* 19 */ "VMCALL with non-clear VMCS.",
503	/* 20 */ "VMCALL with invalid VM-exit control fields.",
504	/* 21 */ "(Not Used)",
505	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
506	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
507	/* 24 */ "VMCALL with invalid SMM-monitor features.",
508	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
509	/* 26 */ "VM entry with events blocked by MOV SS.",
510	/* 27 */ "(Not Used)",
511	/* 28 */ "Invalid operand to INVEPT/INVVPID."
512	};
513	#endif /* VBOX_STRICT */
514
515
516
517	/**
518	* Updates the VM's last error record. If there was a VMX instruction error,
519	* reads the error data from the VMCS and updates VCPU's last error record as
520	* well.
521	*
522	* @param pVM Pointer to the VM.
523	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
524	* VERR_VMX_UNABLE_TO_START_VM or
525	* VERR_VMX_INVALID_VMCS_FIELD).
526	* @param rc The error code.
527	*/
528	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
529	{
530	AssertPtr(pVM);
531	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
532	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
533	{
534	AssertPtrReturnVoid(pVCpu);
535	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
536	}
537	pVM->hm.s.lLastError = rc;
538	}
539
540
541	/**
542	* Reads the VM-entry interruption-information field from the VMCS into the VMX
543	* transient structure.
544	*
545	* @returns VBox status code.
546	* @param pVmxTransient Pointer to the VMX transient structure.
547	*
548	* @remarks No-long-jump zone!!!
549	*/
550	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
551	{
552	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
553	AssertRCReturn(rc, rc);
554	return VINF_SUCCESS;
555	}
556
557
558	/**
559	* Reads the VM-entry exception error code field from the VMCS into
560	* the VMX transient structure.
561	*
562	* @returns VBox status code.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*
565	* @remarks No-long-jump zone!!!
566	*/
567	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
570	AssertRCReturn(rc, rc);
571	return VINF_SUCCESS;
572	}
573
574
575	/**
576	* Reads the VM-entry exception error code field from the VMCS into
577	* the VMX transient structure.
578	*
579	* @returns VBox status code.
580	* @param pVCpu Pointer to the VMCPU.
581	* @param pVmxTransient Pointer to the VMX transient structure.
582	*
583	* @remarks No-long-jump zone!!!
584	*/
585	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
586	{
587	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
588	AssertRCReturn(rc, rc);
589	return VINF_SUCCESS;
590	}
591
592
593	/**
594	* Reads the VM-exit interruption-information field from the VMCS into the VMX
595	* transient structure.
596	*
597	* @returns VBox status code.
598	* @param pVCpu Pointer to the VMCPU.
599	* @param pVmxTransient Pointer to the VMX transient structure.
600	*/
601	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
602	{
603	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
604	{
605	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
606	AssertRCReturn(rc, rc);
607	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
608	}
609	return VINF_SUCCESS;
610	}
611
612
613	/**
614	* Reads the VM-exit interruption error code from the VMCS into the VMX
615	* transient structure.
616	*
617	* @returns VBox status code.
618	* @param pVCpu Pointer to the VMCPU.
619	* @param pVmxTransient Pointer to the VMX transient structure.
620	*/
621	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
622	{
623	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
624	{
625	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
626	AssertRCReturn(rc, rc);
627	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
628	}
629	return VINF_SUCCESS;
630	}
631
632
633	/**
634	* Reads the VM-exit instruction length field from the VMCS into the VMX
635	* transient structure.
636	*
637	* @returns VBox status code.
638	* @param pVCpu Pointer to the VMCPU.
639	* @param pVmxTransient Pointer to the VMX transient structure.
640	*/
641	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
642	{
643	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
644	{
645	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
646	AssertRCReturn(rc, rc);
647	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
648	}
649	return VINF_SUCCESS;
650	}
651
652
653	/**
654	* Reads the VM-exit instruction-information field from the VMCS into
655	* the VMX transient structure.
656	*
657	* @returns VBox status code.
658	* @param pVCpu The cross context per CPU structure.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU.
678	* @param pVmxTransient Pointer to the VMX transient structure.
679	*/
680	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
681	{
682	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
683	{
684	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
685	AssertRCReturn(rc, rc);
686	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
687	}
688	return VINF_SUCCESS;
689	}
690
691
692	/**
693	* Reads the IDT-vectoring information field from the VMCS into the VMX
694	* transient structure.
695	*
696	* @returns VBox status code.
697	* @param pVmxTransient Pointer to the VMX transient structure.
698	*
699	* @remarks No-long-jump zone!!!
700	*/
701	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
702	{
703	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
704	{
705	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
706	AssertRCReturn(rc, rc);
707	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
708	}
709	return VINF_SUCCESS;
710	}
711
712
713	/**
714	* Reads the IDT-vectoring error code from the VMCS into the VMX
715	* transient structure.
716	*
717	* @returns VBox status code.
718	* @param pVmxTransient Pointer to the VMX transient structure.
719	*/
720	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
721	{
722	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
723	{
724	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
725	AssertRCReturn(rc, rc);
726	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
727	}
728	return VINF_SUCCESS;
729	}
730
731
732	/**
733	* Enters VMX root mode operation on the current CPU.
734	*
735	* @returns VBox status code.
736	* @param pVM Pointer to the VM (optional, can be NULL, after
737	* a resume).
738	* @param HCPhysCpuPage Physical address of the VMXON region.
739	* @param pvCpuPage Pointer to the VMXON region.
740	*/
741	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
742	{
743	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
744	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
745	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
746
747	if (pVM)
748	{
749	/* Write the VMCS revision dword to the VMXON region. */
750	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
751	}
752
753	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
754	RTCCUINTREG uEflags = ASMIntDisableFlags();
755
756	/* Enable the VMX bit in CR4 if necessary. */
757	RTCCUINTREG uCr4 = ASMGetCR4();
758	if (!(uCr4 & X86_CR4_VMXE))
759	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
760
761	/* Enter VMX root mode. */
762	int rc = VMXEnable(HCPhysCpuPage);
763	if (RT_FAILURE(rc))
764	ASMSetCR4(uCr4);
765
766	/* Restore interrupts. */
767	ASMSetFlags(uEflags);
768	return rc;
769	}
770
771
772	/**
773	* Exits VMX root mode operation on the current CPU.
774	*
775	* @returns VBox status code.
776	*/
777	static int hmR0VmxLeaveRootMode(void)
778	{
779	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
780
781	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
782	RTCCUINTREG uEflags = ASMIntDisableFlags();
783
784	/* If we're for some reason not in VMX root mode, then don't leave it. */
785	RTCCUINTREG uHostCR4 = ASMGetCR4();
786
787	int rc;
788	if (uHostCR4 & X86_CR4_VMXE)
789	{
790	/* Exit VMX root mode and clear the VMX bit in CR4. */
791	VMXDisable();
792	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
793	rc = VINF_SUCCESS;
794	}
795	else
796	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
797
798	/* Restore interrupts. */
799	ASMSetFlags(uEflags);
800	return rc;
801	}
802
803
804	/**
805	* Allocates and maps one physically contiguous page. The allocated page is
806	* zero'd out. (Used by various VT-x structures).
807	*
808	* @returns IPRT status code.
809	* @param pMemObj Pointer to the ring-0 memory object.
810	* @param ppVirt Where to store the virtual address of the
811	* allocation.
812	* @param pPhys Where to store the physical address of the
813	* allocation.
814	*/
815	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
816	{
817	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
818	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
820
821	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
822	if (RT_FAILURE(rc))
823	return rc;
824	ppVirt = RTR0MemObjAddress(pMemObj);
825	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
826	ASMMemZero32(*ppVirt, PAGE_SIZE);
827	return VINF_SUCCESS;
828	}
829
830
831	/**
832	* Frees and unmaps an allocated physical page.
833	*
834	* @param pMemObj Pointer to the ring-0 memory object.
835	* @param ppVirt Where to re-initialize the virtual address of
836	* allocation as 0.
837	* @param pHCPhys Where to re-initialize the physical address of the
838	* allocation as 0.
839	*/
840	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
841	{
842	AssertPtr(pMemObj);
843	AssertPtr(ppVirt);
844	AssertPtr(pHCPhys);
845	if (*pMemObj != NIL_RTR0MEMOBJ)
846	{
847	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
848	AssertRC(rc);
849	*pMemObj = NIL_RTR0MEMOBJ;
850	*ppVirt = 0;
851	*pHCPhys = 0;
852	}
853	}
854
855
856	/**
857	* Worker function to free VT-x related structures.
858	*
859	* @returns IPRT status code.
860	* @param pVM Pointer to the VM.
861	*/
862	static void hmR0VmxStructsFree(PVM pVM)
863	{
864	for (VMCPUID i = 0; i < pVM->cCpus; i++)
865	{
866	PVMCPU pVCpu = &pVM->aCpus[i];
867	AssertPtr(pVCpu);
868
869	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872	#endif
873
874	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
875	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
876
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
878	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
879	}
880
881	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
882	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
883	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
884	#endif
885	}
886
887
888	/**
889	* Worker function to allocate VT-x related VM structures.
890	*
891	* @returns IPRT status code.
892	* @param pVM Pointer to the VM.
893	*/
894	static int hmR0VmxStructsAlloc(PVM pVM)
895	{
896	/*
897	* Initialize members up-front so we can cleanup properly on allocation failure.
898	*/
899	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
900	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
901	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
902	pVM->hm.s.vmx.HCPhys##a_Name = 0;
903
904	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
905	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
908
909	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
910	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
911	#endif
912	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
913
914	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
915	for (VMCPUID i = 0; i < pVM->cCpus; i++)
916	{
917	PVMCPU pVCpu = &pVM->aCpus[i];
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
921	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
922	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
923	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
924	#endif
925	}
926	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
927	#undef VMXLOCAL_INIT_VM_MEMOBJ
928
929	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
930	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
931	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
932	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
933
934	/*
935	* Allocate all the VT-x structures.
936	*/
937	int rc = VINF_SUCCESS;
938	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
939	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
940	if (RT_FAILURE(rc))
941	goto cleanup;
942	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
943	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
944	#endif
945
946	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
947	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
948	{
949	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
950	&pVM->hm.s.vmx.HCPhysApicAccess);
951	if (RT_FAILURE(rc))
952	goto cleanup;
953	}
954
955	/*
956	* Initialize per-VCPU VT-x structures.
957	*/
958	for (VMCPUID i = 0; i < pVM->cCpus; i++)
959	{
960	PVMCPU pVCpu = &pVM->aCpus[i];
961	AssertPtr(pVCpu);
962
963	/* Allocate the VM control structure (VMCS). */
964	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
965	if (RT_FAILURE(rc))
966	goto cleanup;
967
968	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
969	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
970	{
971	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
972	&pVCpu->hm.s.vmx.HCPhysVirtApic);
973	if (RT_FAILURE(rc))
974	goto cleanup;
975	}
976
977	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
978	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
979	{
980	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
981	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
982	if (RT_FAILURE(rc))
983	goto cleanup;
984	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
985	}
986
987	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
988	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
989	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
990	if (RT_FAILURE(rc))
991	goto cleanup;
992
993	/* Allocate the VM-exit MSR-load page for the host MSRs. */
994	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
995	if (RT_FAILURE(rc))
996	goto cleanup;
997	#endif
998	}
999
1000	return VINF_SUCCESS;
1001
1002	cleanup:
1003	hmR0VmxStructsFree(pVM);
1004	return rc;
1005	}
1006
1007
1008	/**
1009	* Does global VT-x initialization (called during module initialization).
1010	*
1011	* @returns VBox status code.
1012	*/
1013	VMMR0DECL(int) VMXR0GlobalInit(void)
1014	{
1015	#ifdef HMVMX_USE_FUNCTION_TABLE
1016	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1017	# ifdef VBOX_STRICT
1018	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1019	Assert(g_apfnVMExitHandlers[i]);
1020	# endif
1021	#endif
1022	return VINF_SUCCESS;
1023	}
1024
1025
1026	/**
1027	* Does global VT-x termination (called during module termination).
1028	*/
1029	VMMR0DECL(void) VMXR0GlobalTerm()
1030	{
1031	/* Nothing to do currently. */
1032	}
1033
1034
1035	/**
1036	* Sets up and activates VT-x on the current CPU.
1037	*
1038	* @returns VBox status code.
1039	* @param pCpu Pointer to the global CPU info struct.
1040	* @param pVM Pointer to the VM (can be NULL after a host resume
1041	* operation).
1042	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1043	* fEnabledByHost is true).
1044	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1045	* @a fEnabledByHost is true).
1046	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1047	* enable VT-x on the host.
1048	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1049	*/
1050	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1051	void *pvMsrs)
1052	{
1053	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1054	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1055	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1056
1057	/* Enable VT-x if it's not already enabled by the host. */
1058	if (!fEnabledByHost)
1059	{
1060	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1061	if (RT_FAILURE(rc))
1062	return rc;
1063	}
1064
1065	/*
1066	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1067	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1068	*/
1069	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1070	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1071	{
1072	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1073	pCpu->fFlushAsidBeforeUse = false;
1074	}
1075	else
1076	pCpu->fFlushAsidBeforeUse = true;
1077
1078	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1079	++pCpu->cTlbFlushes;
1080
1081	return VINF_SUCCESS;
1082	}
1083
1084
1085	/**
1086	* Deactivates VT-x on the current CPU.
1087	*
1088	* @returns VBox status code.
1089	* @param pCpu Pointer to the global CPU info struct.
1090	* @param pvCpuPage Pointer to the VMXON region.
1091	* @param HCPhysCpuPage Physical address of the VMXON region.
1092	*
1093	* @remarks This function should never be called when SUPR0EnableVTx() or
1094	* similar was used to enable VT-x on the host.
1095	*/
1096	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1097	{
1098	NOREF(pCpu);
1099	NOREF(pvCpuPage);
1100	NOREF(HCPhysCpuPage);
1101
1102	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1103	return hmR0VmxLeaveRootMode();
1104	}
1105
1106
1107	/**
1108	* Sets the permission bits for the specified MSR in the MSR bitmap.
1109	*
1110	* @param pVCpu Pointer to the VMCPU.
1111	* @param uMSR The MSR value.
1112	* @param enmRead Whether reading this MSR causes a VM-exit.
1113	* @param enmWrite Whether writing this MSR causes a VM-exit.
1114	*/
1115	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1116	{
1117	int32_t iBit;
1118	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1119
1120	/*
1121	* Layout:
1122	* 0x000 - 0x3ff - Low MSR read bits
1123	* 0x400 - 0x7ff - High MSR read bits
1124	* 0x800 - 0xbff - Low MSR write bits
1125	* 0xc00 - 0xfff - High MSR write bits
1126	*/
1127	if (uMsr <= 0x00001FFF)
1128	iBit = uMsr;
1129	else if ( uMsr >= 0xC0000000
1130	&& uMsr <= 0xC0001FFF)
1131	{
1132	iBit = (uMsr - 0xC0000000);
1133	pbMsrBitmap += 0x400;
1134	}
1135	else
1136	{
1137	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1138	return;
1139	}
1140
1141	Assert(iBit <= 0x1fff);
1142	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1143	ASMBitSet(pbMsrBitmap, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap, iBit);
1146
1147	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1148	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1149	else
1150	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1151	}
1152
1153
1154	/**
1155	* Flushes the TLB using EPT.
1156	*
1157	* @returns VBox status code.
1158	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1159	* enmFlush).
1160	* @param enmFlush Type of flush.
1161	*
1162	* @remarks Caller is responsible for making sure this function is called only
1163	* when NestedPaging is supported and providing @a enmFlush that is
1164	* supported by the CPU.
1165	* @remarks Can be called with interrupts disabled.
1166	*/
1167	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1168	{
1169	uint64_t au64Descriptor[2];
1170	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1171	au64Descriptor[0] = 0;
1172	else
1173	{
1174	Assert(pVCpu);
1175	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1176	}
1177	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1178
1179	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1180	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1181	rc));
1182	if ( RT_SUCCESS(rc)
1183	&& pVCpu)
1184	{
1185	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1186	}
1187	}
1188
1189
1190	/**
1191	* Flushes the TLB using VPID.
1192	*
1193	* @returns VBox status code.
1194	* @param pVM Pointer to the VM.
1195	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1196	* enmFlush).
1197	* @param enmFlush Type of flush.
1198	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1199	* on @a enmFlush).
1200	*
1201	* @remarks Can be called with interrupts disabled.
1202	*/
1203	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1204	{
1205	AssertPtr(pVM);
1206	Assert(pVM->hm.s.vmx.fVpid);
1207
1208	uint64_t au64Descriptor[2];
1209	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1210	{
1211	au64Descriptor[0] = 0;
1212	au64Descriptor[1] = 0;
1213	}
1214	else
1215	{
1216	AssertPtr(pVCpu);
1217	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1218	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1219	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1220	au64Descriptor[1] = GCPtr;
1221	}
1222
1223	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1224	AssertMsg(rc == VINF_SUCCESS,
1225	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1226	if ( RT_SUCCESS(rc)
1227	&& pVCpu)
1228	{
1229	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1230	}
1231	}
1232
1233
1234	/**
1235	* Invalidates a guest page by guest virtual address. Only relevant for
1236	* EPT/VPID, otherwise there is nothing really to invalidate.
1237	*
1238	* @returns VBox status code.
1239	* @param pVM Pointer to the VM.
1240	* @param pVCpu Pointer to the VMCPU.
1241	* @param GCVirt Guest virtual address of the page to invalidate.
1242	*/
1243	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1244	{
1245	AssertPtr(pVM);
1246	AssertPtr(pVCpu);
1247	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1248
1249	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1250	if (!fFlushPending)
1251	{
1252	/*
1253	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1254	* See @bugref{6043} and @bugref{6177}.
1255	*
1256	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1257	* function maybe called in a loop with individual addresses.
1258	*/
1259	if (pVM->hm.s.vmx.fVpid)
1260	{
1261	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1262	{
1263	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1264	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1265	}
1266	else
1267	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1268	}
1269	else if (pVM->hm.s.fNestedPaging)
1270	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1271	}
1272
1273	return VINF_SUCCESS;
1274	}
1275
1276
1277	/**
1278	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1279	* otherwise there is nothing really to invalidate.
1280	*
1281	* @returns VBox status code.
1282	* @param pVM Pointer to the VM.
1283	* @param pVCpu Pointer to the VMCPU.
1284	* @param GCPhys Guest physical address of the page to invalidate.
1285	*/
1286	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1287	{
1288	LogFlowFunc(("%RGp\n", GCPhys));
1289
1290	/*
1291	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1292	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1293	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1294	*/
1295	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1296	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1297	return VINF_SUCCESS;
1298	}
1299
1300
1301	/**
1302	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1303	* case where neither EPT nor VPID is supported by the CPU.
1304	*
1305	* @param pVM Pointer to the VM.
1306	* @param pVCpu Pointer to the VMCPU.
1307	* @param pCpu Pointer to the global HM struct.
1308	*
1309	* @remarks Called with interrupts disabled.
1310	*/
1311	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1312	{
1313	AssertPtr(pVCpu);
1314	AssertPtr(pCpu);
1315	NOREF(pVM);
1316
1317	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1318
1319	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1320	#if 0
1321	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1322	pVCpu->hm.s.TlbShootdown.cPages = 0;
1323	#endif
1324
1325	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1326	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1327	pVCpu->hm.s.fForceTLBFlush = false;
1328	return;
1329	}
1330
1331
1332	/**
1333	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1334	*
1335	* @param pVM Pointer to the VM.
1336	* @param pVCpu Pointer to the VMCPU.
1337	* @param pCpu Pointer to the global HM CPU struct.
1338	* @remarks All references to "ASID" in this function pertains to "VPID" in
1339	* Intel's nomenclature. The reason is, to avoid confusion in compare
1340	* statements since the host-CPU copies are named "ASID".
1341	*
1342	* @remarks Called with interrupts disabled.
1343	*/
1344	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1345	{
1346	#ifdef VBOX_WITH_STATISTICS
1347	bool fTlbFlushed = false;
1348	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1349	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1350	if (!fTlbFlushed) \
1351	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1352	} while (0)
1353	#else
1354	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1355	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1356	#endif
1357
1358	AssertPtr(pVM);
1359	AssertPtr(pCpu);
1360	AssertPtr(pVCpu);
1361	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1362	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1363	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1364
1365
1366	/*
1367	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1368	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1369	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1370	*/
1371	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1372	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1373	{
1374	++pCpu->uCurrentAsid;
1375	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1376	{
1377	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1378	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1379	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1380	}
1381
1382	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1383	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1384	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1385
1386	/*
1387	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1388	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1389	*/
1390	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1391	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1392	HMVMX_SET_TAGGED_TLB_FLUSHED();
1393	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1394	}
1395
1396	/* Check for explicit TLB shootdowns. */
1397	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1398	{
1399	/*
1400	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1401	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1402	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1403	* but not guest-physical mappings.
1404	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1405	*/
1406	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1407	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1408	HMVMX_SET_TAGGED_TLB_FLUSHED();
1409	}
1410
1411	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1412	* where it is commented out. Support individual entry flushing
1413	* someday. */
1414	#if 0
1415	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1416	{
1417	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1418
1419	/*
1420	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1421	* as supported by the CPU.
1422	*/
1423	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1424	{
1425	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1426	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1427	}
1428	else
1429	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1430
1431	HMVMX_SET_TAGGED_TLB_FLUSHED();
1432	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1433	pVCpu->hm.s.TlbShootdown.cPages = 0;
1434	}
1435	#endif
1436
1437	pVCpu->hm.s.fForceTLBFlush = false;
1438
1439	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1440
1441	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1442	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1443	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1444	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1445	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1446	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1447	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1448	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1449
1450	/* Update VMCS with the VPID. */
1451	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1452	AssertRC(rc);
1453
1454	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1455	}
1456
1457
1458	/**
1459	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1460	*
1461	* @returns VBox status code.
1462	* @param pVM Pointer to the VM.
1463	* @param pVCpu Pointer to the VMCPU.
1464	* @param pCpu Pointer to the global HM CPU struct.
1465	*
1466	* @remarks Called with interrupts disabled.
1467	*/
1468	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1469	{
1470	AssertPtr(pVM);
1471	AssertPtr(pVCpu);
1472	AssertPtr(pCpu);
1473	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1474	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1475
1476	/*
1477	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1478	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1479	*/
1480	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1481	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1482	{
1483	pVCpu->hm.s.fForceTLBFlush = true;
1484	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1485	}
1486
1487	/* Check for explicit TLB shootdown flushes. */
1488	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1489	{
1490	pVCpu->hm.s.fForceTLBFlush = true;
1491	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1492	}
1493
1494	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1495	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1496
1497	if (pVCpu->hm.s.fForceTLBFlush)
1498	{
1499	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1500	pVCpu->hm.s.fForceTLBFlush = false;
1501	}
1502	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1503	* where it is commented out. Support individual entry flushing
1504	* someday. */
1505	#if 0
1506	else
1507	{
1508	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1509	{
1510	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1511	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1512	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1513	}
1514	else
1515	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1516
1517	pVCpu->hm.s.TlbShootdown.cPages = 0;
1518	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1519	}
1520	#endif
1521	}
1522
1523
1524	/**
1525	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1526	*
1527	* @returns VBox status code.
1528	* @param pVM Pointer to the VM.
1529	* @param pVCpu Pointer to the VMCPU.
1530	* @param pCpu Pointer to the global HM CPU struct.
1531	*
1532	* @remarks Called with interrupts disabled.
1533	*/
1534	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1535	{
1536	AssertPtr(pVM);
1537	AssertPtr(pVCpu);
1538	AssertPtr(pCpu);
1539	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1540	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1541
1542	/*
1543	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1544	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1545	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1546	*/
1547	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1548	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1549	{
1550	pVCpu->hm.s.fForceTLBFlush = true;
1551	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1552	}
1553
1554	/* Check for explicit TLB shootdown flushes. */
1555	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1556	{
1557	/*
1558	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1559	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1560	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1561	*/
1562	pVCpu->hm.s.fForceTLBFlush = true;
1563	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1564	}
1565
1566	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1567	if (pVCpu->hm.s.fForceTLBFlush)
1568	{
1569	++pCpu->uCurrentAsid;
1570	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1571	{
1572	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1573	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1574	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1575	}
1576
1577	pVCpu->hm.s.fForceTLBFlush = false;
1578	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1579	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1580	if (pCpu->fFlushAsidBeforeUse)
1581	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1582	}
1583	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1584	* where it is commented out. Support individual entry flushing
1585	* someday. */
1586	#if 0
1587	else
1588	{
1589	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1590	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1591	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1592	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1593
1594	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1595	{
1596	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1597	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1598	{
1599	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1600	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1601	}
1602	else
1603	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1604
1605	pVCpu->hm.s.TlbShootdown.cPages = 0;
1606	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1607	}
1608	else
1609	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1610	}
1611	#endif
1612
1613	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1614	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1615	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1616	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1617	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1618	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1619
1620	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1621	AssertRC(rc);
1622	}
1623
1624
1625	/**
1626	* Flushes the guest TLB entry based on CPU capabilities.
1627	*
1628	* @param pVCpu Pointer to the VMCPU.
1629	* @param pCpu Pointer to the global HM CPU struct.
1630	*/
1631	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1632	{
1633	PVM pVM = pVCpu->CTX_SUFF(pVM);
1634	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1635	{
1636	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1637	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1638	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1639	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1640	default:
1641	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1642	break;
1643	}
1644	}
1645
1646
1647	/**
1648	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1649	* TLB entries from the host TLB before VM-entry.
1650	*
1651	* @returns VBox status code.
1652	* @param pVM Pointer to the VM.
1653	*/
1654	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1655	{
1656	/*
1657	* Determine optimal flush type for Nested Paging.
1658	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1659	* guest execution (see hmR3InitFinalizeR0()).
1660	*/
1661	if (pVM->hm.s.fNestedPaging)
1662	{
1663	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1664	{
1665	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1666	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1667	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1668	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1669	else
1670	{
1671	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1672	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1673	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1674	}
1675
1676	/* Make sure the write-back cacheable memory type for EPT is supported. */
1677	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1678	{
1679	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1680	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1682	}
1683	}
1684	else
1685	{
1686	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1687	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1688	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1689	}
1690	}
1691
1692	/*
1693	* Determine optimal flush type for VPID.
1694	*/
1695	if (pVM->hm.s.vmx.fVpid)
1696	{
1697	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1698	{
1699	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1700	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1701	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1702	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1703	else
1704	{
1705	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1706	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1707	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1708	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1709	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1710	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1711	pVM->hm.s.vmx.fVpid = false;
1712	}
1713	}
1714	else
1715	{
1716	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1717	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1718	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1719	pVM->hm.s.vmx.fVpid = false;
1720	}
1721	}
1722
1723	/*
1724	* Setup the handler for flushing tagged-TLBs.
1725	*/
1726	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1727	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1728	else if (pVM->hm.s.fNestedPaging)
1729	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1730	else if (pVM->hm.s.vmx.fVpid)
1731	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1732	else
1733	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1734	return VINF_SUCCESS;
1735	}
1736
1737
1738	/**
1739	* Sets up pin-based VM-execution controls in the VMCS.
1740	*
1741	* @returns VBox status code.
1742	* @param pVM Pointer to the VM.
1743	* @param pVCpu Pointer to the VMCPU.
1744	*/
1745	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1746	{
1747	AssertPtr(pVM);
1748	AssertPtr(pVCpu);
1749
1750	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1751	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1752
1753	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1754	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1755	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1756
1757	/* Enable the VMX preemption timer. */
1758	if (pVM->hm.s.vmx.fUsePreemptTimer)
1759	{
1760	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1761	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1762	}
1763
1764	if ((val & zap) != val)
1765	{
1766	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1767	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1768	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1769	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1770	}
1771
1772	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1773	AssertRCReturn(rc, rc);
1774
1775	/* Update VCPU with the currently set pin-based VM-execution controls. */
1776	pVCpu->hm.s.vmx.u32PinCtls = val;
1777	return rc;
1778	}
1779
1780
1781	/**
1782	* Sets up processor-based VM-execution controls in the VMCS.
1783	*
1784	* @returns VBox status code.
1785	* @param pVM Pointer to the VM.
1786	* @param pVMCPU Pointer to the VMCPU.
1787	*/
1788	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1789	{
1790	AssertPtr(pVM);
1791	AssertPtr(pVCpu);
1792
1793	int rc = VERR_INTERNAL_ERROR_5;
1794	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1795	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1796
1797	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1798	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1799	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1800	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1801	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1802	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1803	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1804
1805	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1806	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1807	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1808	{
1809	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1810	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1811	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1812	}
1813
1814	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1815	if (!pVM->hm.s.fNestedPaging)
1816	{
1817	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1818	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1819	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1820	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1821	}
1822
1823	/* Use TPR shadowing if supported by the CPU. */
1824	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1825	{
1826	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1827	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1828	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1830	AssertRCReturn(rc, rc);
1831
1832	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1833	/* CR8 writes causes a VM-exit based on TPR threshold. */
1834	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1835	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1836	}
1837	else
1838	{
1839	/*
1840	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
1841	* Set this control only for 64-bit guests.
1842	*/
1843	if (pVM->hm.s.fAllow64BitGuests)
1844	{
1845	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1846	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1847	}
1848	}
1849
1850	/* Use MSR-bitmaps if supported by the CPU. */
1851	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1852	{
1853	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1854
1855	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1856	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1857	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1858	AssertRCReturn(rc, rc);
1859
1860	/*
1861	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1862	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1863	*/
1864	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1865	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1866	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1867	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1868	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1869	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1870	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1871	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1872	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1873	}
1874
1875	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1876	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1877	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1878
1879	if ((val & zap) != val)
1880	{
1881	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1882	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1883	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1885	}
1886
1887	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1888	AssertRCReturn(rc, rc);
1889
1890	/* Update VCPU with the currently set processor-based VM-execution controls. */
1891	pVCpu->hm.s.vmx.u32ProcCtls = val;
1892
1893	/*
1894	* Secondary processor-based VM-execution controls.
1895	*/
1896	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1897	{
1898	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1899	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1900
1901	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1902	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1903
1904	if (pVM->hm.s.fNestedPaging)
1905	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1906	else
1907	{
1908	/*
1909	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1910	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1911	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1912	*/
1913	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1914	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1915	}
1916
1917	if (pVM->hm.s.vmx.fVpid)
1918	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1919
1920	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1921	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1922
1923	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1924	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1925	* done dynamically. */
1926	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1927	{
1928	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1929	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1930	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1931	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1932	AssertRCReturn(rc, rc);
1933	}
1934
1935	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1936	{
1937	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1938	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1939	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1940	}
1941
1942	if ((val & zap) != val)
1943	{
1944	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1945	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1946	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1947	}
1948
1949	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1950	AssertRCReturn(rc, rc);
1951
1952	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1953	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1954	}
1955	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1956	{
1957	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1958	"available\n"));
1959	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1960	}
1961
1962	return VINF_SUCCESS;
1963	}
1964
1965
1966	/**
1967	* Sets up miscellaneous (everything other than Pin & Processor-based
1968	* VM-execution) control fields in the VMCS.
1969	*
1970	* @returns VBox status code.
1971	* @param pVM Pointer to the VM.
1972	* @param pVCpu Pointer to the VMCPU.
1973	*/
1974	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1975	{
1976	AssertPtr(pVM);
1977	AssertPtr(pVCpu);
1978
1979	int rc = VERR_GENERAL_FAILURE;
1980
1981	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1982	#if 0
1983	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
1984	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1985	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1986
1987	/*
1988	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1989	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1990	* We thus use the exception bitmap to control it rather than use both.
1991	*/
1992	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1993	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1994
1995	/** @todo Explore possibility of using IO-bitmaps. */
1996	/* All IO & IOIO instructions cause VM-exits. */
1997	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1998	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1999
2000	/* Initialize the MSR-bitmap area. */
2001	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2002	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2003	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2004	#endif
2005
2006	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2007	/* Setup MSR autoloading/storing. */
2008	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2009	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2010	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2011	AssertRCReturn(rc, rc);
2012	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2013	AssertRCReturn(rc, rc);
2014
2015	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2016	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2017	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2018	AssertRCReturn(rc, rc);
2019	#endif
2020
2021	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2022	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2023	AssertRCReturn(rc, rc);
2024
2025	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2026	#if 0
2027	/* Setup debug controls */
2028	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2029	AssertRCReturn(rc, rc);
2030	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2031	AssertRCReturn(rc, rc);
2032	#endif
2033
2034	return rc;
2035	}
2036
2037
2038	/**
2039	* Sets up the initial exception bitmap in the VMCS based on static conditions
2040	* (i.e. conditions that cannot ever change after starting the VM).
2041	*
2042	* @returns VBox status code.
2043	* @param pVM Pointer to the VM.
2044	* @param pVCpu Pointer to the VMCPU.
2045	*/
2046	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2047	{
2048	AssertPtr(pVM);
2049	AssertPtr(pVCpu);
2050
2051	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2052
2053	uint32_t u32XcptBitmap = 0;
2054
2055	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2056	if (!pVM->hm.s.fNestedPaging)
2057	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2058
2059	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2060	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2061	AssertRCReturn(rc, rc);
2062	return rc;
2063	}
2064
2065
2066	/**
2067	* Sets up the initial guest-state mask. The guest-state mask is consulted
2068	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2069	* for the nested virtualization case (as it would cause a VM-exit).
2070	*
2071	* @param pVCpu Pointer to the VMCPU.
2072	*/
2073	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2074	{
2075	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2076	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2077	return VINF_SUCCESS;
2078	}
2079
2080
2081	/**
2082	* Does per-VM VT-x initialization.
2083	*
2084	* @returns VBox status code.
2085	* @param pVM Pointer to the VM.
2086	*/
2087	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2088	{
2089	LogFlowFunc(("pVM=%p\n", pVM));
2090
2091	int rc = hmR0VmxStructsAlloc(pVM);
2092	if (RT_FAILURE(rc))
2093	{
2094	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2095	return rc;
2096	}
2097
2098	return VINF_SUCCESS;
2099	}
2100
2101
2102	/**
2103	* Does per-VM VT-x termination.
2104	*
2105	* @returns VBox status code.
2106	* @param pVM Pointer to the VM.
2107	*/
2108	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2109	{
2110	LogFlowFunc(("pVM=%p\n", pVM));
2111
2112	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2113	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2114	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2115	#endif
2116	hmR0VmxStructsFree(pVM);
2117	return VINF_SUCCESS;
2118	}
2119
2120
2121	/**
2122	* Sets up the VM for execution under VT-x.
2123	* This function is only called once per-VM during initialization.
2124	*
2125	* @returns VBox status code.
2126	* @param pVM Pointer to the VM.
2127	*/
2128	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2129	{
2130	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2131	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2132
2133	LogFlowFunc(("pVM=%p\n", pVM));
2134
2135	/*
2136	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2137	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2138	*/
2139	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2140	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2141	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2142	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2143	{
2144	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2145	return VERR_INTERNAL_ERROR;
2146	}
2147
2148	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2149	/*
2150	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2151	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2152	*/
2153	if ( pVM->hm.s.fAllow64BitGuests
2154	&& !HMVMX_IS_64BIT_HOST_MODE())
2155	{
2156	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2157	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2158	}
2159	#endif
2160
2161	/* Initialize these always, see hmR3InitFinalizeR0().*/
2162	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2164
2165	/* Setup the tagged-TLB flush handlers. */
2166	int rc = hmR0VmxSetupTaggedTlb(pVM);
2167	if (RT_FAILURE(rc))
2168	{
2169	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2170	return rc;
2171	}
2172
2173	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2174	{
2175	PVMCPU pVCpu = &pVM->aCpus[i];
2176	AssertPtr(pVCpu);
2177	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2178
2179	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2180	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2181
2182	/* Set revision dword at the beginning of the VMCS structure. */
2183	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2184
2185	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2186	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2187	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2188	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2189
2190	/* Load this VMCS as the current VMCS. */
2191	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2192	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2193	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2194
2195	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2196	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2197	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2198
2199	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2200	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2201	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2202
2203	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2204	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2205	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2206
2207	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2208	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2209	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2210
2211	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2212	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2213	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2214
2215	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2216	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2217	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2218	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2219	#endif
2220
2221	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2222	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2223	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2224	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2225
2226	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2227
2228	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2229	}
2230
2231	return VINF_SUCCESS;
2232	}
2233
2234
2235	/**
2236	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2237	* the VMCS.
2238	*
2239	* @returns VBox status code.
2240	* @param pVM Pointer to the VM.
2241	* @param pVCpu Pointer to the VMCPU.
2242	*/
2243	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2244	{
2245	RTCCUINTREG uReg = ASMGetCR0();
2246	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2247	AssertRCReturn(rc, rc);
2248
2249	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2250	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2251	if (HMVMX_IS_64BIT_HOST_MODE())
2252	{
2253	uint64_t uRegCR3 = HMR0Get64bitCR3();
2254	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2255	}
2256	else
2257	#endif
2258	{
2259	uReg = ASMGetCR3();
2260	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2261	}
2262	AssertRCReturn(rc, rc);
2263
2264	uReg = ASMGetCR4();
2265	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2266	AssertRCReturn(rc, rc);
2267	return rc;
2268	}
2269
2270
2271	#if HC_ARCH_BITS == 64
2272	/**
2273	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2274	* requirements. See hmR0VmxSaveHostSegmentRegs().
2275	*/
2276	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2277	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2278	{ \
2279	bool fValidSelector = true; \
2280	if ((selValue) & X86_SEL_LDT) \
2281	{ \
2282	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2283	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2284	} \
2285	if (fValidSelector) \
2286	{ \
2287	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2288	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2289	} \
2290	(selValue) = 0; \
2291	}
2292	#endif
2293
2294
2295	/**
2296	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2297	* the host-state area in the VMCS.
2298	*
2299	* @returns VBox status code.
2300	* @param pVM Pointer to the VM.
2301	* @param pVCpu Pointer to the VMCPU.
2302	*/
2303	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2304	{
2305	int rc = VERR_INTERNAL_ERROR_5;
2306
2307	/*
2308	* Host DS, ES, FS and GS segment registers.
2309	*/
2310	#if HC_ARCH_BITS == 64
2311	RTSEL uSelDS = ASMGetDS();
2312	RTSEL uSelES = ASMGetES();
2313	RTSEL uSelFS = ASMGetFS();
2314	RTSEL uSelGS = ASMGetGS();
2315	#else
2316	RTSEL uSelDS = 0;
2317	RTSEL uSelES = 0;
2318	RTSEL uSelFS = 0;
2319	RTSEL uSelGS = 0;
2320	#endif
2321
2322	/* Recalculate which host-state bits need to be manually restored. */
2323	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2324
2325	/*
2326	* Host CS and SS segment registers.
2327	*/
2328	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2329	RTSEL uSelCS;
2330	RTSEL uSelSS;
2331	if (HMVMX_IS_64BIT_HOST_MODE())
2332	{
2333	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2334	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2335	}
2336	else
2337	{
2338	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2339	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2340	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2341	}
2342	#else
2343	RTSEL uSelCS = ASMGetCS();
2344	RTSEL uSelSS = ASMGetSS();
2345	#endif
2346
2347	/*
2348	* Host TR segment register.
2349	*/
2350	RTSEL uSelTR = ASMGetTR();
2351
2352	#if HC_ARCH_BITS == 64
2353	/*
2354	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2355	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2356	*/
2357	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2358	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2359	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2360	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2361	# undef VMXLOCAL_ADJUST_HOST_SEG
2362	#endif
2363
2364	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2365	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2366	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2367	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2368	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2369	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2370	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2371	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2372	Assert(uSelCS);
2373	Assert(uSelTR);
2374
2375	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2376	#if 0
2377	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2378	Assert(uSelSS != 0);
2379	#endif
2380
2381	/* Write these host selector fields into the host-state area in the VMCS. */
2382	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2383	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2384	#if HC_ARCH_BITS == 64
2385	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2386	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2387	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2388	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2389	#endif
2390	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2391
2392	/*
2393	* Host GDTR and IDTR.
2394	*/
2395	RTGDTR Gdtr;
2396	RT_ZERO(Gdtr);
2397	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2398	if (HMVMX_IS_64BIT_HOST_MODE())
2399	{
2400	X86XDTR64 Gdtr64;
2401	X86XDTR64 Idtr64;
2402	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2403	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2404	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2405
2406	Gdtr.cbGdt = Gdtr64.cb;
2407	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2408	}
2409	else
2410	#endif
2411	{
2412	RTIDTR Idtr;
2413	ASMGetGDTR(&Gdtr);
2414	ASMGetIDTR(&Idtr);
2415	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2416	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2417
2418	#if HC_ARCH_BITS == 64
2419	/*
2420	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2421	* maximum limit (0xffff) on every VM-exit.
2422	*/
2423	if (Gdtr.cbGdt != 0xffff)
2424	{
2425	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2426	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2427	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2428	}
2429
2430	/*
2431	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2432	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2433	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2434	*/
2435	if (Idtr.cbIdt < 0x0fff)
2436	{
2437	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2438	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2439	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2440	}
2441	#endif
2442	}
2443
2444	/*
2445	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2446	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2447	*/
2448	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2449	{
2450	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2451	return VERR_VMX_INVALID_HOST_STATE;
2452	}
2453
2454	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2455	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2456	if (HMVMX_IS_64BIT_HOST_MODE())
2457	{
2458	/* We need the 64-bit TR base for hybrid darwin. */
2459	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2460	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2461	}
2462	else
2463	#endif
2464	{
2465	uintptr_t uTRBase;
2466	#if HC_ARCH_BITS == 64
2467	uTRBase = X86DESC64_BASE(pDesc);
2468
2469	/*
2470	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2471	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2472	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2473	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2474	*
2475	* [1] See Intel spec. 3.5 "System Descriptor Types".
2476	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2477	*/
2478	Assert(pDesc->System.u4Type == 11);
2479	if ( pDesc->System.u16LimitLow != 0x67
2480	\|\| pDesc->System.u4LimitHigh)
2481	{
2482	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2483	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2484
2485	/* Store the GDTR here as we need it while restoring TR. */
2486	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2487	}
2488	#else
2489	uTRBase = X86DESC_BASE(pDesc);
2490	#endif
2491	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2492	}
2493	AssertRCReturn(rc, rc);
2494
2495	/*
2496	* Host FS base and GS base.
2497	*/
2498	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2499	if (HMVMX_IS_64BIT_HOST_MODE())
2500	{
2501	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2502	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2503	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2504	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2505
2506	# if HC_ARCH_BITS == 64
2507	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2508	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2509	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2510	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2511	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2512	# endif
2513	}
2514	#endif
2515	return rc;
2516	}
2517
2518
2519	/**
2520	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2521	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2522	* the host after every successful VM exit.
2523	*
2524	* @returns VBox status code.
2525	* @param pVM Pointer to the VM.
2526	* @param pVCpu Pointer to the VMCPU.
2527	*/
2528	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2529	{
2530	AssertPtr(pVCpu);
2531	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2532
2533	int rc = VINF_SUCCESS;
2534	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2535	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2536	uint32_t cHostMsrs = 0;
2537	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2538
2539	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2540	{
2541	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2542
2543	# if HC_ARCH_BITS == 64
2544	/* Paranoia. 64-bit code requires these bits to be set always. */
2545	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2546
2547	/*
2548	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2549	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2550	* some reason (e.g. allow transparent reads) we would activate the code below.
2551	*/
2552	# if 0
2553	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2554	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2555	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2556	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2557	if (CPUMIsGuestInLongMode(pVCpu))
2558	{
2559	uint64_t u64GuestEfer;
2560	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2561	AssertRC(rc);
2562
2563	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2564	{
2565	pHostMsr->u32Msr = MSR_K6_EFER;
2566	pHostMsr->u32Reserved = 0;
2567	pHostMsr->u64Value = u64HostEfer;
2568	pHostMsr++; cHostMsrs++;
2569	}
2570	}
2571	# endif
2572	# else /* HC_ARCH_BITS != 64 */
2573	pHostMsr->u32Msr = MSR_K6_EFER;
2574	pHostMsr->u32Reserved = 0;
2575	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2576	if (CPUMIsGuestInLongMode(pVCpu))
2577	{
2578	/* Must match the EFER value in our 64 bits switcher. */
2579	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2580	}
2581	else
2582	# endif
2583	pHostMsr->u64Value = u64HostEfer;
2584	pHostMsr++; cHostMsrs++;
2585	# endif /* HC_ARCH_BITS == 64 */
2586	}
2587
2588	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2589	if (HMVMX_IS_64BIT_HOST_MODE())
2590	{
2591	pHostMsr->u32Msr = MSR_K6_STAR;
2592	pHostMsr->u32Reserved = 0;
2593	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2594	pHostMsr++; cHostMsrs++;
2595	pHostMsr->u32Msr = MSR_K8_LSTAR;
2596	pHostMsr->u32Reserved = 0;
2597	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2598	pHostMsr++; cHostMsrs++;
2599	pHostMsr->u32Msr = MSR_K8_SF_MASK;
2600	pHostMsr->u32Reserved = 0;
2601	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2602	pHostMsr++; cHostMsrs++;
2603	pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2604	pHostMsr->u32Reserved = 0;
2605	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2606	pHostMsr++; cHostMsrs++;
2607	}
2608	# endif
2609
2610	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2611	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2612	{
2613	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2614	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2615	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2616	}
2617
2618	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2619	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2620
2621	/*
2622	* Host Sysenter MSRs.
2623	*/
2624	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2625	AssertRCReturn(rc, rc);
2626	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2627	if (HMVMX_IS_64BIT_HOST_MODE())
2628	{
2629	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2630	AssertRCReturn(rc, rc);
2631	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2632	}
2633	else
2634	{
2635	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2636	AssertRCReturn(rc, rc);
2637	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2638	}
2639	#elif HC_ARCH_BITS == 32
2640	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2641	AssertRCReturn(rc, rc);
2642	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2643	#else
2644	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2645	AssertRCReturn(rc, rc);
2646	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2647	#endif
2648	AssertRCReturn(rc, rc);
2649
2650	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2651	* hmR0VmxSetupExitCtls() !! */
2652	return rc;
2653	}
2654
2655
2656	/**
2657	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2658	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2659	* controls".
2660	*
2661	* @returns VBox status code.
2662	* @param pVCpu Pointer to the VMCPU.
2663	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2664	* out-of-sync. Make sure to update the required fields
2665	* before using them.
2666	*
2667	* @remarks No-long-jump zone!!!
2668	*/
2669	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2670	{
2671	int rc = VINF_SUCCESS;
2672	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2673	{
2674	PVM pVM = pVCpu->CTX_SUFF(pVM);
2675	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2676	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2677
2678	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2679	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2680
2681	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2682	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2683	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2684	else
2685	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2686
2687	/*
2688	* The following should -not- be set (since we're not in SMM mode):
2689	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2690	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2691	*/
2692
2693	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2694	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2695	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2696
2697	if ((val & zap) != val)
2698	{
2699	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2700	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2701	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2702	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2703	}
2704
2705	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2706	AssertRCReturn(rc, rc);
2707
2708	/* Update VCPU with the currently set VM-exit controls. */
2709	pVCpu->hm.s.vmx.u32EntryCtls = val;
2710	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2711	}
2712	return rc;
2713	}
2714
2715
2716	/**
2717	* Sets up the VM-exit controls in the VMCS.
2718	*
2719	* @returns VBox status code.
2720	* @param pVM Pointer to the VM.
2721	* @param pVCpu Pointer to the VMCPU.
2722	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2723	* out-of-sync. Make sure to update the required fields
2724	* before using them.
2725	*
2726	* @remarks requires EFER.
2727	*/
2728	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2729	{
2730	int rc = VINF_SUCCESS;
2731	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2732	{
2733	PVM pVM = pVCpu->CTX_SUFF(pVM);
2734	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2735	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2736
2737	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2738	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2739
2740	/*
2741	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2742	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2743	*/
2744	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2745	if (HMVMX_IS_64BIT_HOST_MODE())
2746	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2747	else
2748	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2749	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2750	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2751	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2752	else
2753	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2754	#endif
2755
2756	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2757	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2758
2759	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2760	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2761	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2762	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2763	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2764
2765	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2766	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2767
2768	if ((val & zap) != val)
2769	{
2770	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2771	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2772	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2773	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2774	}
2775
2776	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2777	AssertRCReturn(rc, rc);
2778
2779	/* Update VCPU with the currently set VM-exit controls. */
2780	pVCpu->hm.s.vmx.u32ExitCtls = val;
2781	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2782	}
2783	return rc;
2784	}
2785
2786
2787	/**
2788	* Loads the guest APIC and related state.
2789	*
2790	* @returns VBox status code.
2791	* @param pVM Pointer to the VM.
2792	* @param pVCpu Pointer to the VMCPU.
2793	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2794	* out-of-sync. Make sure to update the required fields
2795	* before using them.
2796	*/
2797	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2798	{
2799	int rc = VINF_SUCCESS;
2800	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2801	{
2802	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2803	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2804	{
2805	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2806
2807	bool fPendingIntr = false;
2808	uint8_t u8Tpr = 0;
2809	uint8_t u8PendingIntr = 0;
2810	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2811	AssertRCReturn(rc, rc);
2812
2813	/*
2814	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2815	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2816	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2817	* the interrupt when we VM-exit for other reasons.
2818	*/
2819	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2820	uint32_t u32TprThreshold = 0;
2821	if (fPendingIntr)
2822	{
2823	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2824	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2825	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2826	if (u8PendingPriority <= u8TprPriority)
2827	u32TprThreshold = u8PendingPriority;
2828	else
2829	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2830	}
2831	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2832
2833	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2834	AssertRCReturn(rc, rc);
2835	}
2836
2837	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
2838	}
2839	return rc;
2840	}
2841
2842
2843	/**
2844	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2845	*
2846	* @returns Guest's interruptibility-state.
2847	* @param pVCpu Pointer to the VMCPU.
2848	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2849	* out-of-sync. Make sure to update the required fields
2850	* before using them.
2851	*
2852	* @remarks No-long-jump zone!!!
2853	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2854	*/
2855	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2856	{
2857	/*
2858	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2859	* inhibit interrupts or clear any existing interrupt-inhibition.
2860	*/
2861	uint32_t uIntrState = 0;
2862	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2863	{
2864	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2865	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2866	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2867	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2868	{
2869	/*
2870	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2871	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2872	*/
2873	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2874	}
2875	else if (pMixedCtx->eflags.Bits.u1IF)
2876	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2877	else
2878	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2879	}
2880	return uIntrState;
2881	}
2882
2883
2884	/**
2885	* Loads the guest's interruptibility-state into the guest-state area in the
2886	* VMCS.
2887	*
2888	* @returns VBox status code.
2889	* @param pVCpu Pointer to the VMCPU.
2890	* @param uIntrState The interruptibility-state to set.
2891	*/
2892	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2893	{
2894	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2895	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2896	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2897	AssertRCReturn(rc, rc);
2898	return rc;
2899	}
2900
2901
2902	/**
2903	* Loads the guest's RIP into the guest-state area in the VMCS.
2904	*
2905	* @returns VBox status code.
2906	* @param pVCpu Pointer to the VMCPU.
2907	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2908	* out-of-sync. Make sure to update the required fields
2909	* before using them.
2910	*
2911	* @remarks No-long-jump zone!!!
2912	*/
2913	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2914	{
2915	int rc = VINF_SUCCESS;
2916	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
2917	{
2918	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2919	AssertRCReturn(rc, rc);
2920
2921	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
2922	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
2923	}
2924	return rc;
2925	}
2926
2927
2928	/**
2929	* Loads the guest's RSP into the guest-state area in the VMCS.
2930	*
2931	* @returns VBox status code.
2932	* @param pVCpu Pointer to the VMCPU.
2933	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2934	* out-of-sync. Make sure to update the required fields
2935	* before using them.
2936	*
2937	* @remarks No-long-jump zone!!!
2938	*/
2939	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2940	{
2941	int rc = VINF_SUCCESS;
2942	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
2943	{
2944	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2945	AssertRCReturn(rc, rc);
2946
2947	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
2948	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2949	}
2950	return rc;
2951	}
2952
2953
2954	/**
2955	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2956	*
2957	* @returns VBox status code.
2958	* @param pVCpu Pointer to the VMCPU.
2959	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2960	* out-of-sync. Make sure to update the required fields
2961	* before using them.
2962	*
2963	* @remarks No-long-jump zone!!!
2964	*/
2965	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2966	{
2967	int rc = VINF_SUCCESS;
2968	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
2969	{
2970	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2971	Let us assert it as such and use 32-bit VMWRITE. */
2972	Assert(!(pMixedCtx->rflags.u64 >> 32));
2973	X86EFLAGS Eflags = pMixedCtx->eflags;
2974	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2975	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2976
2977	/*
2978	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2979	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2980	*/
2981	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2982	{
2983	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2984	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2985	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2986	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2987	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2988	}
2989
2990	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2991	AssertRCReturn(rc, rc);
2992
2993	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
2994	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2995	}
2996	return rc;
2997	}
2998
2999
3000	/**
3001	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3002	*
3003	* @returns VBox status code.
3004	* @param pVCpu Pointer to the VMCPU.
3005	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3006	* out-of-sync. Make sure to update the required fields
3007	* before using them.
3008	*
3009	* @remarks No-long-jump zone!!!
3010	*/
3011	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3012	{
3013	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3014	AssertRCReturn(rc, rc);
3015	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3016	AssertRCReturn(rc, rc);
3017	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3018	AssertRCReturn(rc, rc);
3019	return rc;
3020	}
3021
3022
3023	/**
3024	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3025	* CR0 is partially shared with the host and we have to consider the FPU bits.
3026	*
3027	* @returns VBox status code.
3028	* @param pVM Pointer to the VM.
3029	* @param pVCpu Pointer to the VMCPU.
3030	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3031	* out-of-sync. Make sure to update the required fields
3032	* before using them.
3033	*
3034	* @remarks No-long-jump zone!!!
3035	*/
3036	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3037	{
3038	/*
3039	* Guest CR0.
3040	* Guest FPU.
3041	*/
3042	int rc = VINF_SUCCESS;
3043	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3044	{
3045	Assert(!(pMixedCtx->cr0 >> 32));
3046	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3047	PVM pVM = pVCpu->CTX_SUFF(pVM);
3048
3049	/* The guest's view (read access) of its CR0 is unblemished. */
3050	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3051	AssertRCReturn(rc, rc);
3052	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3053
3054	/* Setup VT-x's view of the guest CR0. */
3055	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3056	if (pVM->hm.s.fNestedPaging)
3057	{
3058	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3059	{
3060	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3061	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3062	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3063	}
3064	else
3065	{
3066	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3067	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3068	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3069	}
3070
3071	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3072	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3073	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3074
3075	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3076	AssertRCReturn(rc, rc);
3077	}
3078	else
3079	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3080
3081	/*
3082	* Guest FPU bits.
3083	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3084	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3085	*/
3086	u32GuestCR0 \|= X86_CR0_NE;
3087	bool fInterceptNM = false;
3088	if (CPUMIsGuestFPUStateActive(pVCpu))
3089	{
3090	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3091	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3092	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3093	}
3094	else
3095	{
3096	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3097	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3098	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3099	}
3100
3101	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3102	bool fInterceptMF = false;
3103	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3104	fInterceptMF = true;
3105
3106	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3107	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3108	{
3109	Assert(PDMVmmDevHeapIsEnabled(pVM));
3110	Assert(pVM->hm.s.vmx.pRealModeTSS);
3111	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3112	fInterceptNM = true;
3113	fInterceptMF = true;
3114	}
3115	else
3116	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3117
3118	if (fInterceptNM)
3119	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3120	else
3121	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3122
3123	if (fInterceptMF)
3124	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3125	else
3126	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3127
3128	/* Additional intercepts for debugging, define these yourself explicitly. */
3129	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3130	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3131	\| RT_BIT(X86_XCPT_BP)
3132	\| RT_BIT(X86_XCPT_DB)
3133	\| RT_BIT(X86_XCPT_DE)
3134	\| RT_BIT(X86_XCPT_NM)
3135	\| RT_BIT(X86_XCPT_UD)
3136	\| RT_BIT(X86_XCPT_NP)
3137	\| RT_BIT(X86_XCPT_SS)
3138	\| RT_BIT(X86_XCPT_GP)
3139	\| RT_BIT(X86_XCPT_PF)
3140	\| RT_BIT(X86_XCPT_MF)
3141	;
3142	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3143	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3144	#endif
3145
3146	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3147
3148	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3149	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3150	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3151	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3152	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3153	else
3154	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3155
3156	u32GuestCR0 \|= uSetCR0;
3157	u32GuestCR0 &= uZapCR0;
3158	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3159
3160	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3161	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3162	AssertRCReturn(rc, rc);
3163	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3164	AssertRCReturn(rc, rc);
3165	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3166
3167	/*
3168	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3169	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3170	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3171	*/
3172	uint32_t u32CR0Mask = 0;
3173	u32CR0Mask = X86_CR0_PE
3174	\| X86_CR0_NE
3175	\| X86_CR0_WP
3176	\| X86_CR0_PG
3177	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3178	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3179	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3180
3181	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3182	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3183	* and @bugref{6944}. */
3184	#if 0
3185	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3186	u32CR0Mask &= ~X86_CR0_PE;
3187	#endif
3188	if (pVM->hm.s.fNestedPaging)
3189	u32CR0Mask &= ~X86_CR0_WP;
3190
3191	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3192	if (fInterceptNM)
3193	{
3194	u32CR0Mask \|= X86_CR0_TS
3195	\| X86_CR0_MP;
3196	}
3197
3198	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3199	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3200	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3201	AssertRCReturn(rc, rc);
3202	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3203
3204	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3205	}
3206	return rc;
3207	}
3208
3209
3210	/**
3211	* Loads the guest control registers (CR3, CR4) into the guest-state area
3212	* in the VMCS.
3213	*
3214	* @returns VBox status code.
3215	* @param pVM Pointer to the VM.
3216	* @param pVCpu Pointer to the VMCPU.
3217	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3218	* out-of-sync. Make sure to update the required fields
3219	* before using them.
3220	*
3221	* @remarks No-long-jump zone!!!
3222	*/
3223	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3224	{
3225	int rc = VINF_SUCCESS;
3226	PVM pVM = pVCpu->CTX_SUFF(pVM);
3227
3228	/*
3229	* Guest CR2.
3230	* It's always loaded in the assembler code. Nothing to do here.
3231	*/
3232
3233	/*
3234	* Guest CR3.
3235	*/
3236	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3237	{
3238	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3239	if (pVM->hm.s.fNestedPaging)
3240	{
3241	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3242
3243	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3244	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3245	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3246	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3247
3248	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3249	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3250	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3251
3252	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3253	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3254	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3255	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3256
3257	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3258	AssertRCReturn(rc, rc);
3259	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3260
3261	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3262	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3263	{
3264	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3265	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3266	{
3267	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3268	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3269	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3270	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3271	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3272	}
3273
3274	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3275	have Unrestricted Execution to handle the guest when it's not using paging. */
3276	GCPhysGuestCR3 = pMixedCtx->cr3;
3277	}
3278	else
3279	{
3280	/*
3281	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3282	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3283	* EPT takes care of translating it to host-physical addresses.
3284	*/
3285	RTGCPHYS GCPhys;
3286	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3287	Assert(PDMVmmDevHeapIsEnabled(pVM));
3288
3289	/* We obtain it here every time as the guest could have relocated this PCI region. */
3290	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3291	AssertRCReturn(rc, rc);
3292
3293	GCPhysGuestCR3 = GCPhys;
3294	}
3295
3296	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3297	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3298	}
3299	else
3300	{
3301	/* Non-nested paging case, just use the hypervisor's CR3. */
3302	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3303
3304	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3305	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3306	}
3307	AssertRCReturn(rc, rc);
3308
3309	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3310	}
3311
3312	/*
3313	* Guest CR4.
3314	*/
3315	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3316	{
3317	Assert(!(pMixedCtx->cr4 >> 32));
3318	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3319
3320	/* The guest's view of its CR4 is unblemished. */
3321	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3322	AssertRCReturn(rc, rc);
3323	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3324
3325	/* Setup VT-x's view of the guest CR4. */
3326	/*
3327	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3328	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3329	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3330	*/
3331	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3332	{
3333	Assert(pVM->hm.s.vmx.pRealModeTSS);
3334	Assert(PDMVmmDevHeapIsEnabled(pVM));
3335	u32GuestCR4 &= ~X86_CR4_VME;
3336	}
3337
3338	if (pVM->hm.s.fNestedPaging)
3339	{
3340	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3341	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3342	{
3343	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3344	u32GuestCR4 \|= X86_CR4_PSE;
3345	/* Our identity mapping is a 32 bits page directory. */
3346	u32GuestCR4 &= ~X86_CR4_PAE;
3347	}
3348	/* else use guest CR4.*/
3349	}
3350	else
3351	{
3352	/*
3353	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3354	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3355	*/
3356	switch (pVCpu->hm.s.enmShadowMode)
3357	{
3358	case PGMMODE_REAL: /* Real-mode. */
3359	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3360	case PGMMODE_32_BIT: /* 32-bit paging. */
3361	{
3362	u32GuestCR4 &= ~X86_CR4_PAE;
3363	break;
3364	}
3365
3366	case PGMMODE_PAE: /* PAE paging. */
3367	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3368	{
3369	u32GuestCR4 \|= X86_CR4_PAE;
3370	break;
3371	}
3372
3373	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3374	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3375	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3376	break;
3377	#endif
3378	default:
3379	AssertFailed();
3380	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3381	}
3382	}
3383
3384	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3385	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3386	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3387	u32GuestCR4 \|= uSetCR4;
3388	u32GuestCR4 &= uZapCR4;
3389
3390	/* Write VT-x's view of the guest CR4 into the VMCS. */
3391	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3392	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3393	AssertRCReturn(rc, rc);
3394
3395	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3396	uint32_t u32CR4Mask = 0;
3397	u32CR4Mask = X86_CR4_VME
3398	\| X86_CR4_PAE
3399	\| X86_CR4_PGE
3400	\| X86_CR4_PSE
3401	\| X86_CR4_VMXE;
3402	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3403	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3404	AssertRCReturn(rc, rc);
3405
3406	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3407	}
3408	return rc;
3409	}
3410
3411
3412	/**
3413	* Loads the guest debug registers into the guest-state area in the VMCS.
3414	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3415	*
3416	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3417	*
3418	* @returns VBox status code.
3419	* @param pVCpu Pointer to the VMCPU.
3420	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3421	* out-of-sync. Make sure to update the required fields
3422	* before using them.
3423	*
3424	* @remarks No-long-jump zone!!!
3425	*/
3426	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3427	{
3428	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3429	return VINF_SUCCESS;
3430
3431	#ifdef VBOX_STRICT
3432	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3433	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3434	{
3435	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3436	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3437	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3438	}
3439	#endif
3440
3441	int rc;
3442	PVM pVM = pVCpu->CTX_SUFF(pVM);
3443	bool fInterceptDB = false;
3444	bool fInterceptMovDRx = false;
3445	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3446	{
3447	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3448	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3449	{
3450	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3451	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3452	AssertRCReturn(rc, rc);
3453	Assert(fInterceptDB == false);
3454	}
3455	else
3456	{
3457	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3458	pVCpu->hm.s.fClearTrapFlag = true;
3459	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3460	fInterceptDB = true;
3461	}
3462	}
3463
3464	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3465	{
3466	/*
3467	* Use the combined guest and host DRx values found in the hypervisor
3468	* register set because the debugger has breakpoints active or someone
3469	* is single stepping on the host side without a monitor trap flag.
3470	*
3471	* Note! DBGF expects a clean DR6 state before executing guest code.
3472	*/
3473	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3474	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3475	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3476	{
3477	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3478	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3479	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3480	}
3481	else
3482	#endif
3483	if (!CPUMIsHyperDebugStateActive(pVCpu))
3484	{
3485	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3486	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3487	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3488	}
3489
3490	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3491	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3492	AssertRCReturn(rc, rc);
3493
3494	pVCpu->hm.s.fUsingHyperDR7 = true;
3495	fInterceptDB = true;
3496	fInterceptMovDRx = true;
3497	}
3498	else
3499	{
3500	/*
3501	* If the guest has enabled debug registers, we need to load them prior to
3502	* executing guest code so they'll trigger at the right time.
3503	*/
3504	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3505	{
3506	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3507	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3508	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3509	{
3510	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3511	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3512	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3513	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3514	}
3515	else
3516	#endif
3517	if (CPUMIsGuestDebugStateActive(pVCpu))
3518	{
3519	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3520	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3521	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3522	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3523	}
3524	}
3525	/*
3526	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3527	* must intercept #DB in order to maintain a correct DR6 guest value.
3528	*/
3529	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3530	else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3531	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3532	\|\| !CPUMIsGuestDebugStateActive(pVCpu))
3533	#else
3534	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3535	#endif
3536	{
3537	fInterceptMovDRx = true;
3538	fInterceptDB = true;
3539	}
3540
3541	/* Update guest DR7. */
3542	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3543	AssertRCReturn(rc, rc);
3544
3545	pVCpu->hm.s.fUsingHyperDR7 = false;
3546	}
3547
3548	/*
3549	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3550	*/
3551	if (fInterceptDB)
3552	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3553	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3554	{
3555	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3556	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3557	#endif
3558	}
3559	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3560	AssertRCReturn(rc, rc);
3561
3562	/*
3563	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3564	*/
3565	if (fInterceptMovDRx)
3566	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3567	else
3568	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3569	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3570	AssertRCReturn(rc, rc);
3571
3572	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3573	return VINF_SUCCESS;
3574	}
3575
3576
3577	#ifdef VBOX_STRICT
3578	/**
3579	* Strict function to validate segment registers.
3580	*
3581	* @remarks ASSUMES CR0 is up to date.
3582	*/
3583	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3584	{
3585	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3586	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3587	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3588	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3589	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3590	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3591	{
3592	/* Protected mode checks */
3593	/* CS */
3594	Assert(pCtx->cs.Attr.n.u1Present);
3595	Assert(!(pCtx->cs.Attr.u & 0xf00));
3596	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3597	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3598	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3599	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3600	\|\| (pCtx->cs.Attr.n.u1Granularity));
3601	/* CS cannot be loaded with NULL in protected mode. */
3602	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3603	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3604	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3605	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3606	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3607	else
3608	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3609	/* SS */
3610	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3611	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3612	if ( !(pCtx->cr0 & X86_CR0_PE)
3613	\|\| pCtx->cs.Attr.n.u4Type == 3)
3614	{
3615	Assert(!pCtx->ss.Attr.n.u2Dpl);
3616	}
3617	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3618	{
3619	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3620	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3621	Assert(pCtx->ss.Attr.n.u1Present);
3622	Assert(!(pCtx->ss.Attr.u & 0xf00));
3623	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3624	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3625	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3626	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3627	\|\| (pCtx->ss.Attr.n.u1Granularity));
3628	}
3629	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3630	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3631	{
3632	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3633	Assert(pCtx->ds.Attr.n.u1Present);
3634	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3635	Assert(!(pCtx->ds.Attr.u & 0xf00));
3636	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3637	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3638	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3639	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3640	\|\| (pCtx->ds.Attr.n.u1Granularity));
3641	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3642	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3643	}
3644	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3645	{
3646	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3647	Assert(pCtx->es.Attr.n.u1Present);
3648	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3649	Assert(!(pCtx->es.Attr.u & 0xf00));
3650	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3651	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3652	\|\| !(pCtx->es.Attr.n.u1Granularity));
3653	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3654	\|\| (pCtx->es.Attr.n.u1Granularity));
3655	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3656	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3657	}
3658	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3659	{
3660	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3661	Assert(pCtx->fs.Attr.n.u1Present);
3662	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3663	Assert(!(pCtx->fs.Attr.u & 0xf00));
3664	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3665	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3666	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3667	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3668	\|\| (pCtx->fs.Attr.n.u1Granularity));
3669	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3670	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3671	}
3672	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3673	{
3674	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3675	Assert(pCtx->gs.Attr.n.u1Present);
3676	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3677	Assert(!(pCtx->gs.Attr.u & 0xf00));
3678	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3679	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3680	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3681	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3682	\|\| (pCtx->gs.Attr.n.u1Granularity));
3683	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3684	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3685	}
3686	/* 64-bit capable CPUs. */
3687	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3688	Assert(!(pCtx->cs.u64Base >> 32));
3689	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3690	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3691	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3692	# endif
3693	}
3694	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3695	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3696	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3697	{
3698	/* Real and v86 mode checks. */
3699	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3700	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3701	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3702	{
3703	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3704	}
3705	else
3706	{
3707	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3708	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3709	}
3710
3711	/* CS */
3712	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3713	Assert(pCtx->cs.u32Limit == 0xffff);
3714	Assert(u32CSAttr == 0xf3);
3715	/* SS */
3716	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3717	Assert(pCtx->ss.u32Limit == 0xffff);
3718	Assert(u32SSAttr == 0xf3);
3719	/* DS */
3720	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3721	Assert(pCtx->ds.u32Limit == 0xffff);
3722	Assert(u32DSAttr == 0xf3);
3723	/* ES */
3724	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3725	Assert(pCtx->es.u32Limit == 0xffff);
3726	Assert(u32ESAttr == 0xf3);
3727	/* FS */
3728	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3729	Assert(pCtx->fs.u32Limit == 0xffff);
3730	Assert(u32FSAttr == 0xf3);
3731	/* GS */
3732	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3733	Assert(pCtx->gs.u32Limit == 0xffff);
3734	Assert(u32GSAttr == 0xf3);
3735	/* 64-bit capable CPUs. */
3736	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3737	Assert(!(pCtx->cs.u64Base >> 32));
3738	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3739	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3740	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3741	# endif
3742	}
3743	}
3744	#endif /* VBOX_STRICT */
3745
3746
3747	/**
3748	* Writes a guest segment register into the guest-state area in the VMCS.
3749	*
3750	* @returns VBox status code.
3751	* @param pVCpu Pointer to the VMCPU.
3752	* @param idxSel Index of the selector in the VMCS.
3753	* @param idxLimit Index of the segment limit in the VMCS.
3754	* @param idxBase Index of the segment base in the VMCS.
3755	* @param idxAccess Index of the access rights of the segment in the VMCS.
3756	* @param pSelReg Pointer to the segment selector.
3757	* @param pCtx Pointer to the guest-CPU context.
3758	*
3759	* @remarks No-long-jump zone!!!
3760	*/
3761	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3762	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3763	{
3764	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3765	AssertRCReturn(rc, rc);
3766	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3767	AssertRCReturn(rc, rc);
3768	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3769	AssertRCReturn(rc, rc);
3770
3771	uint32_t u32Access = pSelReg->Attr.u;
3772	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3773	{
3774	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3775	u32Access = 0xf3;
3776	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3777	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3778	}
3779	else
3780	{
3781	/*
3782	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3783	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3784	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3785	* loaded in protected-mode have their attribute as 0.
3786	*/
3787	if (!u32Access)
3788	u32Access = X86DESCATTR_UNUSABLE;
3789	}
3790
3791	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3792	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3793	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3794
3795	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3796	AssertRCReturn(rc, rc);
3797	return rc;
3798	}
3799
3800
3801	/**
3802	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3803	* into the guest-state area in the VMCS.
3804	*
3805	* @returns VBox status code.
3806	* @param pVM Pointer to the VM.
3807	* @param pVCPU Pointer to the VMCPU.
3808	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3809	* out-of-sync. Make sure to update the required fields
3810	* before using them.
3811	*
3812	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3813	* @remarks No-long-jump zone!!!
3814	*/
3815	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3816	{
3817	int rc = VERR_INTERNAL_ERROR_5;
3818	PVM pVM = pVCpu->CTX_SUFF(pVM);
3819
3820	/*
3821	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3822	*/
3823	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
3824	{
3825	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3826	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3827	{
3828	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3829	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3830	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3831	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3832	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3833	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3834	}
3835
3836	#ifdef VBOX_WITH_REM
3837	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3838	{
3839	Assert(pVM->hm.s.vmx.pRealModeTSS);
3840	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3841	if ( pVCpu->hm.s.vmx.fWasInRealMode
3842	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3843	{
3844	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3845	in real-mode (e.g. OpenBSD 4.0) */
3846	REMFlushTBs(pVM);
3847	Log4(("Load: Switch to protected mode detected!\n"));
3848	pVCpu->hm.s.vmx.fWasInRealMode = false;
3849	}
3850	}
3851	#endif
3852	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3853	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3854	AssertRCReturn(rc, rc);
3855	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3856	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3857	AssertRCReturn(rc, rc);
3858	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3859	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3860	AssertRCReturn(rc, rc);
3861	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3862	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3863	AssertRCReturn(rc, rc);
3864	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3865	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3866	AssertRCReturn(rc, rc);
3867	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3868	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3869	AssertRCReturn(rc, rc);
3870
3871	#ifdef VBOX_STRICT
3872	/* Validate. */
3873	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3874	#endif
3875
3876	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
3877	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3878	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3879	}
3880
3881	/*
3882	* Guest TR.
3883	*/
3884	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
3885	{
3886	/*
3887	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3888	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3889	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3890	*/
3891	uint16_t u16Sel = 0;
3892	uint32_t u32Limit = 0;
3893	uint64_t u64Base = 0;
3894	uint32_t u32AccessRights = 0;
3895
3896	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3897	{
3898	u16Sel = pMixedCtx->tr.Sel;
3899	u32Limit = pMixedCtx->tr.u32Limit;
3900	u64Base = pMixedCtx->tr.u64Base;
3901	u32AccessRights = pMixedCtx->tr.Attr.u;
3902	}
3903	else
3904	{
3905	Assert(pVM->hm.s.vmx.pRealModeTSS);
3906	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3907
3908	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3909	RTGCPHYS GCPhys;
3910	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3911	AssertRCReturn(rc, rc);
3912
3913	X86DESCATTR DescAttr;
3914	DescAttr.u = 0;
3915	DescAttr.n.u1Present = 1;
3916	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3917
3918	u16Sel = 0;
3919	u32Limit = HM_VTX_TSS_SIZE;
3920	u64Base = GCPhys; /* in real-mode phys = virt. */
3921	u32AccessRights = DescAttr.u;
3922	}
3923
3924	/* Validate. */
3925	Assert(!(u16Sel & RT_BIT(2)));
3926	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3927	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3928	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3929	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3930	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3931	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3932	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3933	Assert( (u32Limit & 0xfff) == 0xfff
3934	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3935	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3936	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3937
3938	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3939	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3940	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3941	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3942
3943	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
3944	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3945	}
3946
3947	/*
3948	* Guest GDTR.
3949	*/
3950	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
3951	{
3952	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3953	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3954
3955	/* Validate. */
3956	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3957
3958	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
3959	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3960	}
3961
3962	/*
3963	* Guest LDTR.
3964	*/
3965	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
3966	{
3967	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3968	uint32_t u32Access = 0;
3969	if (!pMixedCtx->ldtr.Attr.u)
3970	u32Access = X86DESCATTR_UNUSABLE;
3971	else
3972	u32Access = pMixedCtx->ldtr.Attr.u;
3973
3974	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3975	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3976	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3977	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3978
3979	/* Validate. */
3980	if (!(u32Access & X86DESCATTR_UNUSABLE))
3981	{
3982	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3983	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3984	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3985	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3986	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3987	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3988	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3989	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3990	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3991	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3992	}
3993
3994	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
3995	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3996	}
3997
3998	/*
3999	* Guest IDTR.
4000	*/
4001	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4002	{
4003	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4004	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4005
4006	/* Validate. */
4007	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4008
4009	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4010	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4011	}
4012
4013	return VINF_SUCCESS;
4014	}
4015
4016
4017	/**
4018	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4019	* areas. These MSRs will automatically be loaded to the host CPU on every
4020	* successful VM entry and stored from the host CPU on every successful VM exit.
4021	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4022	*
4023	* @returns VBox status code.
4024	* @param pVCpu Pointer to the VMCPU.
4025	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4026	* out-of-sync. Make sure to update the required fields
4027	* before using them.
4028	*
4029	* @remarks No-long-jump zone!!!
4030	*/
4031	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4032	{
4033	AssertPtr(pVCpu);
4034	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4035
4036	/*
4037	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4038	*/
4039	int rc = VINF_SUCCESS;
4040	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4041	{
4042	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4043	PVM pVM = pVCpu->CTX_SUFF(pVM);
4044	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4045	uint32_t cGuestMsrs = 0;
4046
4047	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4048	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4049	* when the guest really is in 64-bit mode. */
4050	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4051	if (fSupportsLongMode)
4052	{
4053	pGuestMsr->u32Msr = MSR_K8_LSTAR;
4054	pGuestMsr->u32Reserved = 0;
4055	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4056	pGuestMsr++; cGuestMsrs++;
4057	pGuestMsr->u32Msr = MSR_K6_STAR;
4058	pGuestMsr->u32Reserved = 0;
4059	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4060	pGuestMsr++; cGuestMsrs++;
4061	pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4062	pGuestMsr->u32Reserved = 0;
4063	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4064	pGuestMsr++; cGuestMsrs++;
4065	pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4066	pGuestMsr->u32Reserved = 0;
4067	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4068	pGuestMsr++; cGuestMsrs++;
4069	}
4070
4071	/*
4072	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4073	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
4074	*/
4075	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
4076	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
4077	{
4078	pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4079	pGuestMsr->u32Reserved = 0;
4080	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4081	AssertRCReturn(rc, rc);
4082	pGuestMsr++; cGuestMsrs++;
4083	}
4084
4085	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4086	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4087	{
4088	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4089	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4090	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4091	}
4092
4093	/* Update the VCPU's copy of the guest MSR count. */
4094	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4095	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4096	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4097	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4098
4099	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4100	}
4101
4102	/*
4103	* Guest Sysenter MSRs.
4104	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4105	* VM-exits on WRMSRs for these MSRs.
4106	*/
4107	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4108	{
4109	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4110	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4111	}
4112
4113	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4114	{
4115	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4116	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4117	}
4118
4119	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4120	{
4121	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4122	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4123	}
4124
4125	return rc;
4126	}
4127
4128
4129	/**
4130	* Loads the guest activity state into the guest-state area in the VMCS.
4131	*
4132	* @returns VBox status code.
4133	* @param pVCpu Pointer to the VMCPU.
4134	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4135	* out-of-sync. Make sure to update the required fields
4136	* before using them.
4137	*
4138	* @remarks No-long-jump zone!!!
4139	*/
4140	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4141	{
4142	/** @todo See if we can make use of other states, e.g.
4143	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4144	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4145	{
4146	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4147	AssertRCReturn(rc, rc);
4148
4149	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4150	}
4151	return VINF_SUCCESS;
4152	}
4153
4154
4155	/**
4156	* Sets up the appropriate function to run guest code.
4157	*
4158	* @returns VBox status code.
4159	* @param pVCpu Pointer to the VMCPU.
4160	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4161	* out-of-sync. Make sure to update the required fields
4162	* before using them.
4163	*
4164	* @remarks No-long-jump zone!!!
4165	*/
4166	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4167	{
4168	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4169	{
4170	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4171	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4172	#endif
4173	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4174	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4175	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4176	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4177	{
4178	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4179	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4180	}
4181	#else
4182	/* 64-bit host or hybrid host. */
4183	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4184	#endif
4185	}
4186	else
4187	{
4188	/* Guest is not in long mode, use the 32-bit handler. */
4189	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4190	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4191	{
4192	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4193	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4194	}
4195	#else
4196	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4197	#endif
4198	}
4199	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4200	return VINF_SUCCESS;
4201	}
4202
4203
4204	/**
4205	* Wrapper for running the guest code in VT-x.
4206	*
4207	* @returns VBox strict status code.
4208	* @param pVM Pointer to the VM.
4209	* @param pVCpu Pointer to the VMCPU.
4210	* @param pCtx Pointer to the guest-CPU context.
4211	*
4212	* @remarks No-long-jump zone!!!
4213	*/
4214	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4215	{
4216	/*
4217	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4218	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4219	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4220	*/
4221	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4222	/** @todo Add stats for resume vs launch. */
4223	#ifdef VBOX_WITH_KERNEL_USING_XMM
4224	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4225	#else
4226	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4227	#endif
4228	}
4229
4230
4231	/**
4232	* Reports world-switch error and dumps some useful debug info.
4233	*
4234	* @param pVM Pointer to the VM.
4235	* @param pVCpu Pointer to the VMCPU.
4236	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4237	* @param pCtx Pointer to the guest-CPU context.
4238	* @param pVmxTransient Pointer to the VMX transient structure (only
4239	* exitReason updated).
4240	*/
4241	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4242	{
4243	Assert(pVM);
4244	Assert(pVCpu);
4245	Assert(pCtx);
4246	Assert(pVmxTransient);
4247	HMVMX_ASSERT_PREEMPT_SAFE();
4248
4249	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4250	switch (rcVMRun)
4251	{
4252	case VERR_VMX_INVALID_VMXON_PTR:
4253	AssertFailed();
4254	break;
4255	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4256	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4257	{
4258	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4259	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4260	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4261	AssertRC(rc);
4262
4263	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4264	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4265	Cannot do it here as we may have been long preempted. */
4266
4267	#ifdef VBOX_STRICT
4268	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4269	pVmxTransient->uExitReason));
4270	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4271	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4272	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4273	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4274	else
4275	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4276	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4277	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4278
4279	/* VMX control bits. */
4280	uint32_t u32Val;
4281	uint64_t u64Val;
4282	HMVMXHCUINTREG uHCReg;
4283	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4284	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4285	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4286	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4287	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4288	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4289	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4290	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4291	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4292	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4293	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4294	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4295	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4296	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4297	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4298	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4299	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4300	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4301	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4302	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4303	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4304	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4305	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4306	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4307	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4308	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4309	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4310	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4311	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4312	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4313	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4314	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4315	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4316	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4317	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4318	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4319	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4320	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4321	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4322	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4323	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4324	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4325
4326	/* Guest bits. */
4327	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4328	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4329	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4330	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4331	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4332	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4333	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4334	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4335
4336	/* Host bits. */
4337	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4338	Log4(("Host CR0 %#RHr\n", uHCReg));
4339	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4340	Log4(("Host CR3 %#RHr\n", uHCReg));
4341	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4342	Log4(("Host CR4 %#RHr\n", uHCReg));
4343
4344	RTGDTR HostGdtr;
4345	PCX86DESCHC pDesc;
4346	ASMGetGDTR(&HostGdtr);
4347	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4348	Log4(("Host CS %#08x\n", u32Val));
4349	if (u32Val < HostGdtr.cbGdt)
4350	{
4351	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4352	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4353	}
4354
4355	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4356	Log4(("Host DS %#08x\n", u32Val));
4357	if (u32Val < HostGdtr.cbGdt)
4358	{
4359	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4360	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4361	}
4362
4363	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4364	Log4(("Host ES %#08x\n", u32Val));
4365	if (u32Val < HostGdtr.cbGdt)
4366	{
4367	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4368	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4369	}
4370
4371	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4372	Log4(("Host FS %#08x\n", u32Val));
4373	if (u32Val < HostGdtr.cbGdt)
4374	{
4375	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4376	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4377	}
4378
4379	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4380	Log4(("Host GS %#08x\n", u32Val));
4381	if (u32Val < HostGdtr.cbGdt)
4382	{
4383	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4384	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4385	}
4386
4387	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4388	Log4(("Host SS %#08x\n", u32Val));
4389	if (u32Val < HostGdtr.cbGdt)
4390	{
4391	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4392	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4393	}
4394
4395	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4396	Log4(("Host TR %#08x\n", u32Val));
4397	if (u32Val < HostGdtr.cbGdt)
4398	{
4399	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4400	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4401	}
4402
4403	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4404	Log4(("Host TR Base %#RHv\n", uHCReg));
4405	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4406	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4407	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4408	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4409	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4410	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4411	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4412	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4413	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4414	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4415	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4416	Log4(("Host RSP %#RHv\n", uHCReg));
4417	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4418	Log4(("Host RIP %#RHv\n", uHCReg));
4419	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4420	if (HMVMX_IS_64BIT_HOST_MODE())
4421	{
4422	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4423	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4424	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4425	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4426	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4427	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4428	}
4429	# endif
4430	#endif /* VBOX_STRICT */
4431	break;
4432	}
4433
4434	default:
4435	/* Impossible */
4436	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4437	break;
4438	}
4439	NOREF(pVM);
4440	}
4441
4442
4443	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4444	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4445	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4446	#endif
4447	#ifdef VBOX_STRICT
4448	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4449	{
4450	switch (idxField)
4451	{
4452	case VMX_VMCS_GUEST_RIP:
4453	case VMX_VMCS_GUEST_RSP:
4454	case VMX_VMCS_GUEST_SYSENTER_EIP:
4455	case VMX_VMCS_GUEST_SYSENTER_ESP:
4456	case VMX_VMCS_GUEST_GDTR_BASE:
4457	case VMX_VMCS_GUEST_IDTR_BASE:
4458	case VMX_VMCS_GUEST_CS_BASE:
4459	case VMX_VMCS_GUEST_DS_BASE:
4460	case VMX_VMCS_GUEST_ES_BASE:
4461	case VMX_VMCS_GUEST_FS_BASE:
4462	case VMX_VMCS_GUEST_GS_BASE:
4463	case VMX_VMCS_GUEST_SS_BASE:
4464	case VMX_VMCS_GUEST_LDTR_BASE:
4465	case VMX_VMCS_GUEST_TR_BASE:
4466	case VMX_VMCS_GUEST_CR3:
4467	return true;
4468	}
4469	return false;
4470	}
4471
4472	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4473	{
4474	switch (idxField)
4475	{
4476	/* Read-only fields. */
4477	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4478	return true;
4479	}
4480	/* Remaining readable fields should also be writable. */
4481	return hmR0VmxIsValidWriteField(idxField);
4482	}
4483	#endif /* VBOX_STRICT */
4484
4485
4486	/**
4487	* Executes the specified handler in 64-bit mode.
4488	*
4489	* @returns VBox status code.
4490	* @param pVM Pointer to the VM.
4491	* @param pVCpu Pointer to the VMCPU.
4492	* @param pCtx Pointer to the guest CPU context.
4493	* @param enmOp The operation to perform.
4494	* @param cbParam Number of parameters.
4495	* @param paParam Array of 32-bit parameters.
4496	*/
4497	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4498	uint32_t *paParam)
4499	{
4500	int rc, rc2;
4501	PHMGLOBALCPUINFO pCpu;
4502	RTHCPHYS HCPhysCpuPage;
4503	RTCCUINTREG uOldEflags;
4504
4505	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4506	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4507	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4508	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4509
4510	#ifdef VBOX_STRICT
4511	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4512	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4513
4514	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4515	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4516	#endif
4517
4518	/* Disable interrupts. */
4519	uOldEflags = ASMIntDisableFlags();
4520
4521	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4522	RTCPUID idHostCpu = RTMpCpuId();
4523	CPUMR0SetLApic(pVCpu, idHostCpu);
4524	#endif
4525
4526	pCpu = HMR0GetCurrentCpu();
4527	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4528
4529	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4530	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4531
4532	/* Leave VMX Root Mode. */
4533	VMXDisable();
4534
4535	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4536
4537	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4538	CPUMSetHyperEIP(pVCpu, enmOp);
4539	for (int i = (int)cbParam - 1; i >= 0; i--)
4540	CPUMPushHyper(pVCpu, paParam[i]);
4541
4542	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4543
4544	/* Call the switcher. */
4545	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4546	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4547
4548	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4549	/* Make sure the VMX instructions don't cause #UD faults. */
4550	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4551
4552	/* Re-enter VMX Root Mode */
4553	rc2 = VMXEnable(HCPhysCpuPage);
4554	if (RT_FAILURE(rc2))
4555	{
4556	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4557	ASMSetFlags(uOldEflags);
4558	return rc2;
4559	}
4560
4561	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4562	AssertRC(rc2);
4563	Assert(!(ASMGetFlags() & X86_EFL_IF));
4564	ASMSetFlags(uOldEflags);
4565	return rc;
4566	}
4567
4568
4569	/**
4570	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4571	* supporting 64-bit guests.
4572	*
4573	* @returns VBox status code.
4574	* @param fResume Whether to VMLAUNCH or VMRESUME.
4575	* @param pCtx Pointer to the guest-CPU context.
4576	* @param pCache Pointer to the VMCS cache.
4577	* @param pVM Pointer to the VM.
4578	* @param pVCpu Pointer to the VMCPU.
4579	*/
4580	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4581	{
4582	uint32_t aParam[6];
4583	PHMGLOBALCPUINFO pCpu = NULL;
4584	RTHCPHYS HCPhysCpuPage = 0;
4585	int rc = VERR_INTERNAL_ERROR_5;
4586
4587	pCpu = HMR0GetCurrentCpu();
4588	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4589
4590	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4591	pCache->uPos = 1;
4592	pCache->interPD = PGMGetInterPaeCR3(pVM);
4593	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4594	#endif
4595
4596	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4597	pCache->TestIn.HCPhysCpuPage = 0;
4598	pCache->TestIn.HCPhysVmcs = 0;
4599	pCache->TestIn.pCache = 0;
4600	pCache->TestOut.HCPhysVmcs = 0;
4601	pCache->TestOut.pCache = 0;
4602	pCache->TestOut.pCtx = 0;
4603	pCache->TestOut.eflags = 0;
4604	#endif
4605
4606	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4607	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4608	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4609	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4610	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4611	aParam[5] = 0;
4612
4613	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4614	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4615	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4616	#endif
4617	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4618
4619	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4620	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4621	Assert(pCtx->dr[4] == 10);
4622	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4623	#endif
4624
4625	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4626	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4627	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4628	pVCpu->hm.s.vmx.HCPhysVmcs));
4629	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4630	pCache->TestOut.HCPhysVmcs));
4631	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4632	pCache->TestOut.pCache));
4633	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4634	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4635	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4636	pCache->TestOut.pCtx));
4637	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4638	#endif
4639	return rc;
4640	}
4641
4642
4643	/**
4644	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4645	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4646	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4647	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4648	*
4649	* @returns VBox status code.
4650	* @param pVM Pointer to the VM.
4651	* @param pVCpu Pointer to the VMCPU.
4652	*/
4653	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4654	{
4655	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4656	{ \
4657	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4658	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4659	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4660	++cReadFields; \
4661	}
4662
4663	AssertPtr(pVM);
4664	AssertPtr(pVCpu);
4665	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4666	uint32_t cReadFields = 0;
4667
4668	/*
4669	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4670	* and serve to indicate exceptions to the rules.
4671	*/
4672
4673	/* Guest-natural selector base fields. */
4674	#if 0
4675	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4676	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4677	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4678	#endif
4679	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4680	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4681	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4682	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4683	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4684	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4685	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4686	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4687	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4688	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4689	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4690	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4691	#if 0
4692	/* Unused natural width guest-state fields. */
4693	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4694	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4695	#endif
4696	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4697	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4698
4699	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4700	#if 0
4701	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4702	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4703	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4704	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4705	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4706	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4707	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4708	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4709	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4710	#endif
4711
4712	/* Natural width guest-state fields. */
4713	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4714	#if 0
4715	/* Currently unused field. */
4716	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4717	#endif
4718
4719	if (pVM->hm.s.fNestedPaging)
4720	{
4721	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4722	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4723	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4724	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4725	}
4726	else
4727	{
4728	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4729	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4730	}
4731
4732	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4733	return VINF_SUCCESS;
4734	}
4735
4736
4737	/**
4738	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4739	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4740	* darwin, running 64-bit guests).
4741	*
4742	* @returns VBox status code.
4743	* @param pVCpu Pointer to the VMCPU.
4744	* @param idxField The VMCS field encoding.
4745	* @param u64Val 16, 32 or 64 bits value.
4746	*/
4747	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4748	{
4749	int rc;
4750	switch (idxField)
4751	{
4752	/*
4753	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4754	*/
4755	/* 64-bit Control fields. */
4756	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4757	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4758	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4759	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4760	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4761	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4762	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4763	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4764	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4765	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4766	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4767	case VMX_VMCS64_CTRL_EPTP_FULL:
4768	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4769	/* 64-bit Guest-state fields. */
4770	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4771	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4772	case VMX_VMCS64_GUEST_PAT_FULL:
4773	case VMX_VMCS64_GUEST_EFER_FULL:
4774	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4775	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4776	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4777	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4778	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4779	/* 64-bit Host-state fields. */
4780	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4781	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4782	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4783	{
4784	rc = VMXWriteVmcs32(idxField, u64Val);
4785	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4786	break;
4787	}
4788
4789	/*
4790	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4791	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4792	*/
4793	/* Natural-width Guest-state fields. */
4794	case VMX_VMCS_GUEST_CR3:
4795	case VMX_VMCS_GUEST_ES_BASE:
4796	case VMX_VMCS_GUEST_CS_BASE:
4797	case VMX_VMCS_GUEST_SS_BASE:
4798	case VMX_VMCS_GUEST_DS_BASE:
4799	case VMX_VMCS_GUEST_FS_BASE:
4800	case VMX_VMCS_GUEST_GS_BASE:
4801	case VMX_VMCS_GUEST_LDTR_BASE:
4802	case VMX_VMCS_GUEST_TR_BASE:
4803	case VMX_VMCS_GUEST_GDTR_BASE:
4804	case VMX_VMCS_GUEST_IDTR_BASE:
4805	case VMX_VMCS_GUEST_RSP:
4806	case VMX_VMCS_GUEST_RIP:
4807	case VMX_VMCS_GUEST_SYSENTER_ESP:
4808	case VMX_VMCS_GUEST_SYSENTER_EIP:
4809	{
4810	if (!(u64Val >> 32))
4811	{
4812	/* If this field is 64-bit, VT-x will zero out the top bits. */
4813	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4814	}
4815	else
4816	{
4817	/* Assert that only the 32->64 switcher case should ever come here. */
4818	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4819	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4820	}
4821	break;
4822	}
4823
4824	default:
4825	{
4826	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4827	rc = VERR_INVALID_PARAMETER;
4828	break;
4829	}
4830	}
4831	AssertRCReturn(rc, rc);
4832	return rc;
4833	}
4834
4835
4836	/**
4837	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4838	* hosts (except darwin) for 64-bit guests.
4839	*
4840	* @param pVCpu Pointer to the VMCPU.
4841	* @param idxField The VMCS field encoding.
4842	* @param u64Val 16, 32 or 64 bits value.
4843	*/
4844	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4845	{
4846	AssertPtr(pVCpu);
4847	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4848
4849	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4850	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4851
4852	/* Make sure there are no duplicates. */
4853	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4854	{
4855	if (pCache->Write.aField[i] == idxField)
4856	{
4857	pCache->Write.aFieldVal[i] = u64Val;
4858	return VINF_SUCCESS;
4859	}
4860	}
4861
4862	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4863	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4864	pCache->Write.cValidEntries++;
4865	return VINF_SUCCESS;
4866	}
4867
4868	/* Enable later when the assembly code uses these as callbacks. */
4869	#if 0
4870	/*
4871	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4872	*
4873	* @param pVCpu Pointer to the VMCPU.
4874	* @param pCache Pointer to the VMCS cache.
4875	*
4876	* @remarks No-long-jump zone!!!
4877	*/
4878	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4879	{
4880	AssertPtr(pCache);
4881	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4882	{
4883	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4884	AssertRC(rc);
4885	}
4886	pCache->Write.cValidEntries = 0;
4887	}
4888
4889
4890	/**
4891	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4892	*
4893	* @param pVCpu Pointer to the VMCPU.
4894	* @param pCache Pointer to the VMCS cache.
4895	*
4896	* @remarks No-long-jump zone!!!
4897	*/
4898	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4899	{
4900	AssertPtr(pCache);
4901	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4902	{
4903	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4904	AssertRC(rc);
4905	}
4906	}
4907	#endif
4908	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4909
4910
4911	/**
4912	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4913	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4914	* timer.
4915	*
4916	* @returns VBox status code.
4917	* @param pVCpu Pointer to the VMCPU.
4918	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4919	* out-of-sync. Make sure to update the required fields
4920	* before using them.
4921	* @remarks No-long-jump zone!!!
4922	*/
4923	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4924	{
4925	int rc = VERR_INTERNAL_ERROR_5;
4926	bool fOffsettedTsc = false;
4927	PVM pVM = pVCpu->CTX_SUFF(pVM);
4928	if (pVM->hm.s.vmx.fUsePreemptTimer)
4929	{
4930	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4931
4932	/* Make sure the returned values have sane upper and lower boundaries. */
4933	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4934	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4935	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4936	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4937
4938	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4939	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4940	}
4941	else
4942	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4943
4944	if (fOffsettedTsc)
4945	{
4946	uint64_t u64CurTSC = ASMReadTSC();
4947	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4948	{
4949	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4950	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4951
4952	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4953	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4954	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4955	}
4956	else
4957	{
4958	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4959	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4960	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4961	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4962	}
4963	}
4964	else
4965	{
4966	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4967	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4968	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4969	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4970	}
4971	}
4972
4973
4974	/**
4975	* Determines if an exception is a contributory exception. Contributory
4976	* exceptions are ones which can cause double-faults. Page-fault is
4977	* intentionally not included here as it's a conditional contributory exception.
4978	*
4979	* @returns true if the exception is contributory, false otherwise.
4980	* @param uVector The exception vector.
4981	*/
4982	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4983	{
4984	switch (uVector)
4985	{
4986	case X86_XCPT_GP:
4987	case X86_XCPT_SS:
4988	case X86_XCPT_NP:
4989	case X86_XCPT_TS:
4990	case X86_XCPT_DE:
4991	return true;
4992	default:
4993	break;
4994	}
4995	return false;
4996	}
4997
4998
4999	/**
5000	* Sets an event as a pending event to be injected into the guest.
5001	*
5002	* @param pVCpu Pointer to the VMCPU.
5003	* @param u32IntInfo The VM-entry interruption-information field.
5004	* @param cbInstr The VM-entry instruction length in bytes (for software
5005	* interrupts, exceptions and privileged software
5006	* exceptions).
5007	* @param u32ErrCode The VM-entry exception error code.
5008	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5009	* page-fault.
5010	*
5011	* @remarks Statistics counter assumes this is a guest event being injected or
5012	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5013	* always incremented.
5014	*/
5015	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5016	RTGCUINTPTR GCPtrFaultAddress)
5017	{
5018	Assert(!pVCpu->hm.s.Event.fPending);
5019	pVCpu->hm.s.Event.fPending = true;
5020	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5021	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5022	pVCpu->hm.s.Event.cbInstr = cbInstr;
5023	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5024
5025	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5026	}
5027
5028
5029	/**
5030	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5031	*
5032	* @param pVCpu Pointer to the VMCPU.
5033	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5034	* out-of-sync. Make sure to update the required fields
5035	* before using them.
5036	*/
5037	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5038	{
5039	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5040	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5041	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5042	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5043	}
5044
5045
5046	/**
5047	* Handle a condition that occurred while delivering an event through the guest
5048	* IDT.
5049	*
5050	* @returns VBox status code (informational error codes included).
5051	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5052	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5053	* continue execution of the guest which will delivery the #DF.
5054	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5055	*
5056	* @param pVCpu Pointer to the VMCPU.
5057	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5058	* out-of-sync. Make sure to update the required fields
5059	* before using them.
5060	* @param pVmxTransient Pointer to the VMX transient structure.
5061	*
5062	* @remarks No-long-jump zone!!!
5063	*/
5064	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5065	{
5066	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5067	AssertRCReturn(rc, rc);
5068	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5069	{
5070	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5071	AssertRCReturn(rc, rc);
5072
5073	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5074	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5075	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5076
5077	typedef enum
5078	{
5079	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5080	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5081	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5082	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5083	} VMXREFLECTXCPT;
5084
5085	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5086	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5087	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5088	{
5089	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5090	{
5091	enmReflect = VMXREFLECTXCPT_XCPT;
5092	#ifdef VBOX_STRICT
5093	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5094	&& uExitVector == X86_XCPT_PF)
5095	{
5096	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5097	}
5098	#endif
5099	if ( uExitVector == X86_XCPT_PF
5100	&& uIdtVector == X86_XCPT_PF)
5101	{
5102	pVmxTransient->fVectoringPF = true;
5103	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5104	}
5105	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5106	&& hmR0VmxIsContributoryXcpt(uExitVector)
5107	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5108	\|\| uIdtVector == X86_XCPT_PF))
5109	{
5110	enmReflect = VMXREFLECTXCPT_DF;
5111	}
5112	else if (uIdtVector == X86_XCPT_DF)
5113	enmReflect = VMXREFLECTXCPT_TF;
5114	}
5115	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5116	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5117	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5118	{
5119	/*
5120	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5121	* (whatever they are) as they reoccur when restarting the instruction.
5122	*/
5123	enmReflect = VMXREFLECTXCPT_XCPT;
5124	}
5125	}
5126	else
5127	{
5128	/*
5129	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5130	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5131	* original exception to the guest after handling the VM-exit.
5132	*/
5133	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5134	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5135	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5136	{
5137	enmReflect = VMXREFLECTXCPT_XCPT;
5138	}
5139	}
5140
5141	switch (enmReflect)
5142	{
5143	case VMXREFLECTXCPT_XCPT:
5144	{
5145	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5146	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5147	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5148
5149	uint32_t u32ErrCode = 0;
5150	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5151	{
5152	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5153	AssertRCReturn(rc, rc);
5154	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5155	}
5156
5157	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5158	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5159	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5160	rc = VINF_SUCCESS;
5161	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5162	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5163
5164	break;
5165	}
5166
5167	case VMXREFLECTXCPT_DF:
5168	{
5169	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5170	rc = VINF_HM_DOUBLE_FAULT;
5171	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5172	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5173
5174	break;
5175	}
5176
5177	case VMXREFLECTXCPT_TF:
5178	{
5179	rc = VINF_EM_RESET;
5180	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5181	uExitVector));
5182	break;
5183	}
5184
5185	default:
5186	Assert(rc == VINF_SUCCESS);
5187	break;
5188	}
5189	}
5190	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5191	return rc;
5192	}
5193
5194
5195	/**
5196	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5197	*
5198	* @returns VBox status code.
5199	* @param pVCpu Pointer to the VMCPU.
5200	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5201	* out-of-sync. Make sure to update the required fields
5202	* before using them.
5203	*
5204	* @remarks No-long-jump zone!!!
5205	*/
5206	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5207	{
5208	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5209	{
5210	uint32_t uVal = 0;
5211	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5212	AssertRCReturn(rc, rc);
5213
5214	uint32_t uShadow = 0;
5215	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5216	AssertRCReturn(rc, rc);
5217
5218	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5219	CPUMSetGuestCR0(pVCpu, uVal);
5220	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5221	}
5222	return VINF_SUCCESS;
5223	}
5224
5225
5226	/**
5227	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5228	*
5229	* @returns VBox status code.
5230	* @param pVCpu Pointer to the VMCPU.
5231	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5232	* out-of-sync. Make sure to update the required fields
5233	* before using them.
5234	*
5235	* @remarks No-long-jump zone!!!
5236	*/
5237	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5238	{
5239	int rc = VINF_SUCCESS;
5240	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5241	{
5242	uint32_t uVal = 0;
5243	uint32_t uShadow = 0;
5244	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5245	AssertRCReturn(rc, rc);
5246	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5247	AssertRCReturn(rc, rc);
5248
5249	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5250	CPUMSetGuestCR4(pVCpu, uVal);
5251	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5252	}
5253	return rc;
5254	}
5255
5256
5257	/**
5258	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5259	*
5260	* @returns VBox status code.
5261	* @param pVCpu Pointer to the VMCPU.
5262	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5263	* out-of-sync. Make sure to update the required fields
5264	* before using them.
5265	*
5266	* @remarks No-long-jump zone!!!
5267	*/
5268	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5269	{
5270	int rc = VINF_SUCCESS;
5271	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5272	{
5273	uint64_t u64Val = 0;
5274	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5275	AssertRCReturn(rc, rc);
5276
5277	pMixedCtx->rip = u64Val;
5278	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5279	}
5280	return rc;
5281	}
5282
5283
5284	/**
5285	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5286	*
5287	* @returns VBox status code.
5288	* @param pVCpu Pointer to the VMCPU.
5289	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5290	* out-of-sync. Make sure to update the required fields
5291	* before using them.
5292	*
5293	* @remarks No-long-jump zone!!!
5294	*/
5295	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5296	{
5297	int rc = VINF_SUCCESS;
5298	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5299	{
5300	uint64_t u64Val = 0;
5301	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5302	AssertRCReturn(rc, rc);
5303
5304	pMixedCtx->rsp = u64Val;
5305	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5306	}
5307	return rc;
5308	}
5309
5310
5311	/**
5312	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5313	*
5314	* @returns VBox status code.
5315	* @param pVCpu Pointer to the VMCPU.
5316	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5317	* out-of-sync. Make sure to update the required fields
5318	* before using them.
5319	*
5320	* @remarks No-long-jump zone!!!
5321	*/
5322	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5323	{
5324	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5325	{
5326	uint32_t uVal = 0;
5327	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5328	AssertRCReturn(rc, rc);
5329
5330	pMixedCtx->eflags.u32 = uVal;
5331	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5332	{
5333	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5334	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5335
5336	pMixedCtx->eflags.Bits.u1VM = 0;
5337	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5338	}
5339
5340	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5341	}
5342	return VINF_SUCCESS;
5343	}
5344
5345
5346	/**
5347	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5348	* guest-CPU context.
5349	*/
5350	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5351	{
5352	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5353	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5354	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5355	return rc;
5356	}
5357
5358
5359	/**
5360	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5361	* from the guest-state area in the VMCS.
5362	*
5363	* @param pVCpu Pointer to the VMCPU.
5364	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5365	* out-of-sync. Make sure to update the required fields
5366	* before using them.
5367	*
5368	* @remarks No-long-jump zone!!!
5369	*/
5370	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5371	{
5372	uint32_t uIntrState = 0;
5373	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5374	AssertRC(rc);
5375
5376	if (!uIntrState)
5377	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5378	else
5379	{
5380	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5381	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5382	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5383	AssertRC(rc);
5384	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5385	AssertRC(rc);
5386
5387	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5388	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5389	}
5390	}
5391
5392
5393	/**
5394	* Saves the guest's activity state.
5395	*
5396	* @returns VBox status code.
5397	* @param pVCpu Pointer to the VMCPU.
5398	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5399	* out-of-sync. Make sure to update the required fields
5400	* before using them.
5401	*
5402	* @remarks No-long-jump zone!!!
5403	*/
5404	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5405	{
5406	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5407	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5408	return VINF_SUCCESS;
5409	}
5410
5411
5412	/**
5413	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5414	* the current VMCS into the guest-CPU context.
5415	*
5416	* @returns VBox status code.
5417	* @param pVCpu Pointer to the VMCPU.
5418	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5419	* out-of-sync. Make sure to update the required fields
5420	* before using them.
5421	*
5422	* @remarks No-long-jump zone!!!
5423	*/
5424	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5425	{
5426	int rc = VINF_SUCCESS;
5427	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5428	{
5429	uint32_t u32Val = 0;
5430	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5431	pMixedCtx->SysEnter.cs = u32Val;
5432	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5433	}
5434
5435	uint64_t u64Val = 0;
5436	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5437	{
5438	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5439	pMixedCtx->SysEnter.eip = u64Val;
5440	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5441	}
5442	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5443	{
5444	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5445	pMixedCtx->SysEnter.esp = u64Val;
5446	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5447	}
5448	return rc;
5449	}
5450
5451
5452	/**
5453	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5454	* context.
5455	*
5456	* @returns VBox status code.
5457	* @param pVCpu Pointer to the VMCPU.
5458	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5459	* out-of-sync. Make sure to update the required fields
5460	* before using them.
5461	*
5462	* @remarks No-long-jump zone!!!
5463	*/
5464	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5465	{
5466	int rc = VINF_SUCCESS;
5467	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5468	{
5469	uint64_t u64Val = 0;
5470	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5471	pMixedCtx->fs.u64Base = u64Val;
5472	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5473	}
5474	return rc;
5475	}
5476
5477
5478	/**
5479	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5480	* context.
5481	*
5482	* @returns VBox status code.
5483	* @param pVCpu Pointer to the VMCPU.
5484	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5485	* out-of-sync. Make sure to update the required fields
5486	* before using them.
5487	*
5488	* @remarks No-long-jump zone!!!
5489	*/
5490	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5491	{
5492	int rc = VINF_SUCCESS;
5493	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5494	{
5495	uint64_t u64Val = 0;
5496	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5497	pMixedCtx->gs.u64Base = u64Val;
5498	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5499	}
5500	return rc;
5501	}
5502
5503
5504	/**
5505	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5506	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5507	* and TSC_AUX.
5508	*
5509	* @returns VBox status code.
5510	* @param pVCpu Pointer to the VMCPU.
5511	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5512	* out-of-sync. Make sure to update the required fields
5513	* before using them.
5514	*
5515	* @remarks No-long-jump zone!!!
5516	*/
5517	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5518	{
5519	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5520	return VINF_SUCCESS;
5521
5522	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5523	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5524	{
5525	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5526	pMsr += i;
5527	switch (pMsr->u32Msr)
5528	{
5529	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5530	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5531	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5532	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5533	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5534	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5535	default:
5536	{
5537	AssertFailed();
5538	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5539	}
5540	}
5541	}
5542	#endif
5543
5544	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5545	return VINF_SUCCESS;
5546	}
5547
5548
5549	/**
5550	* Saves the guest control registers from the current VMCS into the guest-CPU
5551	* context.
5552	*
5553	* @returns VBox status code.
5554	* @param pVCpu Pointer to the VMCPU.
5555	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5556	* out-of-sync. Make sure to update the required fields
5557	* before using them.
5558	*
5559	* @remarks No-long-jump zone!!!
5560	*/
5561	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5562	{
5563	/* Guest CR0. Guest FPU. */
5564	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5565	AssertRCReturn(rc, rc);
5566
5567	/* Guest CR4. */
5568	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5569	AssertRCReturn(rc, rc);
5570
5571	/* Guest CR2 - updated always during the world-switch or in #PF. */
5572	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5573	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5574	{
5575	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5576	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5577
5578	PVM pVM = pVCpu->CTX_SUFF(pVM);
5579	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5580	\|\| ( pVM->hm.s.fNestedPaging
5581	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5582	{
5583	uint64_t u64Val = 0;
5584	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5585	if (pMixedCtx->cr3 != u64Val)
5586	{
5587	CPUMSetGuestCR3(pVCpu, u64Val);
5588	if (VMMRZCallRing3IsEnabled(pVCpu))
5589	{
5590	PGMUpdateCR3(pVCpu, u64Val);
5591	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5592	}
5593	else
5594	{
5595	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5596	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5597	}
5598	}
5599
5600	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5601	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5602	{
5603	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5604	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5605	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5606	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5607
5608	if (VMMRZCallRing3IsEnabled(pVCpu))
5609	{
5610	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5611	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5612	}
5613	else
5614	{
5615	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5616	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5617	}
5618	}
5619	}
5620
5621	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5622	}
5623
5624	/*
5625	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5626	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5627	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5628	*
5629	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5630	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5631	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5632	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5633	*
5634	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5635	*/
5636	if (VMMRZCallRing3IsEnabled(pVCpu))
5637	{
5638	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5639	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5640
5641	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5642	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5643
5644	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5645	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5646	}
5647
5648	return rc;
5649	}
5650
5651
5652	/**
5653	* Reads a guest segment register from the current VMCS into the guest-CPU
5654	* context.
5655	*
5656	* @returns VBox status code.
5657	* @param pVCpu Pointer to the VMCPU.
5658	* @param idxSel Index of the selector in the VMCS.
5659	* @param idxLimit Index of the segment limit in the VMCS.
5660	* @param idxBase Index of the segment base in the VMCS.
5661	* @param idxAccess Index of the access rights of the segment in the VMCS.
5662	* @param pSelReg Pointer to the segment selector.
5663	*
5664	* @remarks No-long-jump zone!!!
5665	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5666	* macro as that takes care of whether to read from the VMCS cache or
5667	* not.
5668	*/
5669	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5670	PCPUMSELREG pSelReg)
5671	{
5672	uint32_t u32Val = 0;
5673	int rc = VMXReadVmcs32(idxSel, &u32Val);
5674	AssertRCReturn(rc, rc);
5675	pSelReg->Sel = (uint16_t)u32Val;
5676	pSelReg->ValidSel = (uint16_t)u32Val;
5677	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5678
5679	rc = VMXReadVmcs32(idxLimit, &u32Val);
5680	AssertRCReturn(rc, rc);
5681	pSelReg->u32Limit = u32Val;
5682
5683	uint64_t u64Val = 0;
5684	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5685	AssertRCReturn(rc, rc);
5686	pSelReg->u64Base = u64Val;
5687
5688	rc = VMXReadVmcs32(idxAccess, &u32Val);
5689	AssertRCReturn(rc, rc);
5690	pSelReg->Attr.u = u32Val;
5691
5692	/*
5693	* If VT-x marks the segment as unusable, most other bits remain undefined:
5694	* - For CS the L, D and G bits have meaning.
5695	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5696	* - For the remaining data segments no bits are defined.
5697	*
5698	* The present bit and the unusable bit has been observed to be set at the
5699	* same time (the selector was supposed to invalid as we started executing
5700	* a V8086 interrupt in ring-0).
5701	*
5702	* What should be important for the rest of the VBox code that the P bit is
5703	* cleared. Some of the other VBox code recognizes the unusable bit, but
5704	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5705	* safe side here, we'll strip off P and other bits we don't care about. If
5706	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5707	*
5708	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5709	*/
5710	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5711	{
5712	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5713
5714	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5715	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5716	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5717
5718	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5719	#ifdef DEBUG_bird
5720	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5721	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5722	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5723	#endif
5724	}
5725	return VINF_SUCCESS;
5726	}
5727
5728
5729	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5730	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5731	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5732	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5733	#else
5734	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5735	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5736	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5737	#endif
5738
5739
5740	/**
5741	* Saves the guest segment registers from the current VMCS into the guest-CPU
5742	* context.
5743	*
5744	* @returns VBox status code.
5745	* @param pVCpu Pointer to the VMCPU.
5746	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5747	* out-of-sync. Make sure to update the required fields
5748	* before using them.
5749	*
5750	* @remarks No-long-jump zone!!!
5751	*/
5752	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5753	{
5754	/* Guest segment registers. */
5755	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5756	{
5757	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5758	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5759	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5760	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5761	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5762	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5763	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5764
5765	/* Restore segment attributes for real-on-v86 mode hack. */
5766	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5767	{
5768	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5769	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5770	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5771	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5772	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5773	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5774	}
5775	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5776	}
5777
5778	return VINF_SUCCESS;
5779	}
5780
5781
5782	/**
5783	* Saves the guest descriptor table registers and task register from the current
5784	* VMCS into the guest-CPU context.
5785	*
5786	* @returns VBox status code.
5787	* @param pVCpu Pointer to the VMCPU.
5788	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5789	* out-of-sync. Make sure to update the required fields
5790	* before using them.
5791	*
5792	* @remarks No-long-jump zone!!!
5793	*/
5794	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5795	{
5796	int rc = VINF_SUCCESS;
5797
5798	/* Guest LDTR. */
5799	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5800	{
5801	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5802	AssertRCReturn(rc, rc);
5803	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5804	}
5805
5806	/* Guest GDTR. */
5807	uint64_t u64Val = 0;
5808	uint32_t u32Val = 0;
5809	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5810	{
5811	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5812	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5813	pMixedCtx->gdtr.pGdt = u64Val;
5814	pMixedCtx->gdtr.cbGdt = u32Val;
5815	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5816	}
5817
5818	/* Guest IDTR. */
5819	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5820	{
5821	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5822	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5823	pMixedCtx->idtr.pIdt = u64Val;
5824	pMixedCtx->idtr.cbIdt = u32Val;
5825	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5826	}
5827
5828	/* Guest TR. */
5829	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5830	{
5831	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5832	AssertRCReturn(rc, rc);
5833
5834	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5835	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5836	{
5837	rc = VMXLOCAL_READ_SEG(TR, tr);
5838	AssertRCReturn(rc, rc);
5839	}
5840	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5841	}
5842	return rc;
5843	}
5844
5845	#undef VMXLOCAL_READ_SEG
5846
5847
5848	/**
5849	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5850	* context.
5851	*
5852	* @returns VBox status code.
5853	* @param pVCpu Pointer to the VMCPU.
5854	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5855	* out-of-sync. Make sure to update the required fields
5856	* before using them.
5857	*
5858	* @remarks No-long-jump zone!!!
5859	*/
5860	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5861	{
5862	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5863	{
5864	if (!pVCpu->hm.s.fUsingHyperDR7)
5865	{
5866	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5867	uint32_t u32Val;
5868	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5869	pMixedCtx->dr[7] = u32Val;
5870	}
5871
5872	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5873	}
5874	return VINF_SUCCESS;
5875	}
5876
5877
5878	/**
5879	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5880	*
5881	* @returns VBox status code.
5882	* @param pVCpu Pointer to the VMCPU.
5883	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5884	* out-of-sync. Make sure to update the required fields
5885	* before using them.
5886	*
5887	* @remarks No-long-jump zone!!!
5888	*/
5889	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5890	{
5891	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5892	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5893	return VINF_SUCCESS;
5894	}
5895
5896
5897	/**
5898	* Saves the entire guest state from the currently active VMCS into the
5899	* guest-CPU context. This essentially VMREADs all guest-data.
5900	*
5901	* @returns VBox status code.
5902	* @param pVCpu Pointer to the VMCPU.
5903	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5904	* out-of-sync. Make sure to update the required fields
5905	* before using them.
5906	*/
5907	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5908	{
5909	Assert(pVCpu);
5910	Assert(pMixedCtx);
5911
5912	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5913	return VINF_SUCCESS;
5914
5915	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5916	again on the ring-3 callback path, there is no real need to. */
5917	if (VMMRZCallRing3IsEnabled(pVCpu))
5918	VMMR0LogFlushDisable(pVCpu);
5919	else
5920	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5921	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5922
5923	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5924	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5925
5926	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5927	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5928
5929	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5930	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5931
5932	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5933	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5934
5935	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5936	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5937
5938	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5939	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5940
5941	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5942	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5943
5944	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5945	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5946
5947	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5948	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5949
5950	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5951	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5952
5953	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5954	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5955
5956	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5957	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5958
5959	if (VMMRZCallRing3IsEnabled(pVCpu))
5960	VMMR0LogFlushEnable(pVCpu);
5961
5962	return rc;
5963	}
5964
5965
5966	/**
5967	* Check per-VM and per-VCPU force flag actions that require us to go back to
5968	* ring-3 for one reason or another.
5969	*
5970	* @returns VBox status code (information status code included).
5971	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5972	* ring-3.
5973	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5974	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5975	* interrupts)
5976	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5977	* all EMTs to be in ring-3.
5978	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5979	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5980	* to the EM loop.
5981	*
5982	* @param pVM Pointer to the VM.
5983	* @param pVCpu Pointer to the VMCPU.
5984	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5985	* out-of-sync. Make sure to update the required fields
5986	* before using them.
5987	*/
5988	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5989	{
5990	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5991
5992	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5993	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5994	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5995	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5996	{
5997	/* We need the control registers now, make sure the guest-CPU context is updated. */
5998	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5999	AssertRCReturn(rc3, rc3);
6000
6001	/* Pending HM CR3 sync. */
6002	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6003	{
6004	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6005	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6006	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6007	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6008	}
6009
6010	/* Pending HM PAE PDPEs. */
6011	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6012	{
6013	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6014	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6015	}
6016
6017	/* Pending PGM C3 sync. */
6018	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6019	{
6020	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6021	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6022	if (rc2 != VINF_SUCCESS)
6023	{
6024	AssertRC(rc2);
6025	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6026	return rc2;
6027	}
6028	}
6029
6030	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6031	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6032	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6033	{
6034	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6035	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6036	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6037	return rc2;
6038	}
6039
6040	/* Pending VM request packets, such as hardware interrupts. */
6041	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6042	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6043	{
6044	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6045	return VINF_EM_PENDING_REQUEST;
6046	}
6047
6048	/* Pending PGM pool flushes. */
6049	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6050	{
6051	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6052	return VINF_PGM_POOL_FLUSH_PENDING;
6053	}
6054
6055	/* Pending DMA requests. */
6056	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6057	{
6058	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6059	return VINF_EM_RAW_TO_R3;
6060	}
6061	}
6062
6063	return VINF_SUCCESS;
6064	}
6065
6066
6067	/**
6068	* Converts any TRPM trap into a pending HM event. This is typically used when
6069	* entering from ring-3 (not longjmp returns).
6070	*
6071	* @param pVCpu Pointer to the VMCPU.
6072	*/
6073	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6074	{
6075	Assert(TRPMHasTrap(pVCpu));
6076	Assert(!pVCpu->hm.s.Event.fPending);
6077
6078	uint8_t uVector;
6079	TRPMEVENT enmTrpmEvent;
6080	RTGCUINT uErrCode;
6081	RTGCUINTPTR GCPtrFaultAddress;
6082	uint8_t cbInstr;
6083
6084	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6085	AssertRC(rc);
6086
6087	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6088	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6089	if (enmTrpmEvent == TRPM_TRAP)
6090	{
6091	switch (uVector)
6092	{
6093	case X86_XCPT_BP:
6094	case X86_XCPT_OF:
6095	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6096	break;
6097
6098	case X86_XCPT_PF:
6099	case X86_XCPT_DF:
6100	case X86_XCPT_TS:
6101	case X86_XCPT_NP:
6102	case X86_XCPT_SS:
6103	case X86_XCPT_GP:
6104	case X86_XCPT_AC:
6105	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6106	/* no break! */
6107	default:
6108	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6109	break;
6110	}
6111	}
6112	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6113	{
6114	if (uVector == X86_XCPT_NMI)
6115	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6116	else
6117	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6118	}
6119	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6120	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6121	else
6122	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6123
6124	rc = TRPMResetTrap(pVCpu);
6125	AssertRC(rc);
6126	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6127	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6128
6129	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6130	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6131	}
6132
6133
6134	/**
6135	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6136	* VT-x to execute any instruction.
6137	*
6138	* @param pvCpu Pointer to the VMCPU.
6139	*/
6140	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6141	{
6142	Assert(pVCpu->hm.s.Event.fPending);
6143
6144	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6145	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6146	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6147	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6148
6149	/* If a trap was already pending, we did something wrong! */
6150	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6151
6152	TRPMEVENT enmTrapType;
6153	switch (uVectorType)
6154	{
6155	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6156	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6157	enmTrapType = TRPM_HARDWARE_INT;
6158	break;
6159
6160	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6161	enmTrapType = TRPM_SOFTWARE_INT;
6162	break;
6163
6164	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6165	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6166	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6167	enmTrapType = TRPM_TRAP;
6168	break;
6169
6170	default:
6171	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6172	enmTrapType = TRPM_32BIT_HACK;
6173	break;
6174	}
6175
6176	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6177
6178	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6179	AssertRC(rc);
6180
6181	if (fErrorCodeValid)
6182	TRPMSetErrorCode(pVCpu, uErrorCode);
6183
6184	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6185	&& uVector == X86_XCPT_PF)
6186	{
6187	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6188	}
6189	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6190	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6191	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6192	{
6193	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6194	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6195	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6196	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6197	}
6198	pVCpu->hm.s.Event.fPending = false;
6199	}
6200
6201
6202	/**
6203	* Does the necessary state syncing before returning to ring-3 for any reason
6204	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6205	*
6206	* @returns VBox status code.
6207	* @param pVM Pointer to the VM.
6208	* @param pVCpu Pointer to the VMCPU.
6209	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6210	* be out-of-sync. Make sure to update the required
6211	* fields before using them.
6212	* @param fSaveGuestState Whether to save the guest state or not.
6213	*
6214	* @remarks If you modify code here, make sure to check whether
6215	* hmR0VmxCallRing3Callback() needs to be updated too.
6216	* @remarks No-long-jmp zone!!!
6217	*/
6218	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6219	{
6220	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6221	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6222
6223	RTCPUID idCpu = RTMpCpuId();
6224	Log4Func(("HostCpuId=%u\n", idCpu));
6225
6226	/* Save the guest state if necessary. */
6227	if ( fSaveGuestState
6228	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6229	{
6230	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6231	AssertRCReturn(rc, rc);
6232	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6233	}
6234
6235	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6236	if (CPUMIsGuestFPUStateActive(pVCpu))
6237	{
6238	/* We shouldn't reload CR0 without saving it first. */
6239	if (!fSaveGuestState)
6240	{
6241	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6242	AssertRCReturn(rc, rc);
6243	}
6244	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6245	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6246	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6247	}
6248
6249	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6250	#ifdef VBOX_STRICT
6251	if (CPUMIsHyperDebugStateActive(pVCpu))
6252	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6253	#endif
6254	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6255	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6256	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6257	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6258
6259	#if HC_ARCH_BITS == 64
6260	/* Restore host-state bits that VT-x only restores partially. */
6261	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6262	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6263	{
6264	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6265	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6266	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6267	}
6268	#endif
6269
6270	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6271	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6272	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6273	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6274	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6275	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6276	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6277	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6278
6279	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6280
6281	/** @todo This kinda defeats the purpose of having preemption hooks.
6282	* The problem is, deregistering the hooks should be moved to a place that
6283	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6284	* context.
6285	*/
6286	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6287	{
6288	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6289	AssertRCReturn(rc, rc);
6290
6291	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6292	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6293	}
6294	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6295	NOREF(idCpu);
6296
6297	return VINF_SUCCESS;
6298	}
6299
6300
6301	/**
6302	* Leaves the VT-x session.
6303	*
6304	* @returns VBox status code.
6305	* @param pVM Pointer to the VM.
6306	* @param pVCpu Pointer to the VMCPU.
6307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6308	* out-of-sync. Make sure to update the required fields
6309	* before using them.
6310	*
6311	* @remarks No-long-jmp zone!!!
6312	*/
6313	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6314	{
6315	HM_DISABLE_PREEMPT_IF_NEEDED();
6316	HMVMX_ASSERT_CPU_SAFE();
6317	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6318	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6319
6320	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6321	and done this from the VMXR0ThreadCtxCallback(). */
6322	if (!pVCpu->hm.s.fLeaveDone)
6323	{
6324	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6325	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6326	pVCpu->hm.s.fLeaveDone = true;
6327	}
6328
6329	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6330	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6331	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6332	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6333	VMMR0ThreadCtxHooksDeregister(pVCpu);
6334
6335	/* Leave HM context. This takes care of local init (term). */
6336	int rc = HMR0LeaveCpu(pVCpu);
6337
6338	HM_RESTORE_PREEMPT_IF_NEEDED();
6339
6340	return rc;
6341	}
6342
6343
6344	/**
6345	* Does the necessary state syncing before doing a longjmp to ring-3.
6346	*
6347	* @returns VBox status code.
6348	* @param pVM Pointer to the VM.
6349	* @param pVCpu Pointer to the VMCPU.
6350	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6351	* out-of-sync. Make sure to update the required fields
6352	* before using them.
6353	*
6354	* @remarks No-long-jmp zone!!!
6355	*/
6356	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6357	{
6358	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6359	}
6360
6361
6362	/**
6363	* Take necessary actions before going back to ring-3.
6364	*
6365	* An action requires us to go back to ring-3. This function does the necessary
6366	* steps before we can safely return to ring-3. This is not the same as longjmps
6367	* to ring-3, this is voluntary and prepares the guest so it may continue
6368	* executing outside HM (recompiler/IEM).
6369	*
6370	* @returns VBox status code.
6371	* @param pVM Pointer to the VM.
6372	* @param pVCpu Pointer to the VMCPU.
6373	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6374	* out-of-sync. Make sure to update the required fields
6375	* before using them.
6376	* @param rcExit The reason for exiting to ring-3. Can be
6377	* VINF_VMM_UNKNOWN_RING3_CALL.
6378	*/
6379	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6380	{
6381	Assert(pVM);
6382	Assert(pVCpu);
6383	Assert(pMixedCtx);
6384	HMVMX_ASSERT_PREEMPT_SAFE();
6385
6386	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6387	{
6388	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6389	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6390	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6391	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6392	}
6393
6394	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6395	VMMRZCallRing3Disable(pVCpu);
6396	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6397
6398	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6399	if (pVCpu->hm.s.Event.fPending)
6400	{
6401	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6402	Assert(!pVCpu->hm.s.Event.fPending);
6403	}
6404
6405	/* Save guest state and restore host state bits. */
6406	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6407	AssertRCReturn(rc, rc);
6408	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6409
6410	/* Sync recompiler state. */
6411	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6412	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6413	\| CPUM_CHANGED_LDTR
6414	\| CPUM_CHANGED_GDTR
6415	\| CPUM_CHANGED_IDTR
6416	\| CPUM_CHANGED_TR
6417	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6418	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6419	if ( pVM->hm.s.fNestedPaging
6420	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6421	{
6422	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6423	}
6424
6425	/*
6426	* Clear the X86_EFL_TF if necessary.
6427	*/
6428	if (pVCpu->hm.s.fClearTrapFlag)
6429	{
6430	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6431	pMixedCtx->eflags.Bits.u1TF = 0;
6432	pVCpu->hm.s.fClearTrapFlag = false;
6433	}
6434	/** @todo there seems to be issues with the resume flag when the monitor trap
6435	* flag is pending without being used. Seen early in bios init when
6436	* accessing APIC page in prot mode. */
6437
6438	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6439	if (rcExit != VINF_EM_RAW_INTERRUPT)
6440	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6441
6442	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6443
6444	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6445	VMMRZCallRing3RemoveNotification(pVCpu);
6446	VMMRZCallRing3Enable(pVCpu);
6447
6448	return rc;
6449	}
6450
6451
6452	/**
6453	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6454	* longjump to ring-3 and possibly get preempted.
6455	*
6456	* @returns VBox status code.
6457	* @param pVCpu Pointer to the VMCPU.
6458	* @param enmOperation The operation causing the ring-3 longjump.
6459	* @param pvUser Opaque pointer to the guest-CPU context. The data
6460	* may be out-of-sync. Make sure to update the required
6461	* fields before using them.
6462	* @remarks If you modify code here, make sure to check whether
6463	* hmR0VmxLeave() needs to be updated too.
6464	*/
6465	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6466	{
6467	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6468	{
6469	VMMRZCallRing3RemoveNotification(pVCpu);
6470	HM_DISABLE_PREEMPT_IF_NEEDED();
6471
6472	/* If anything here asserts or fails, good luck. */
6473	if (CPUMIsGuestFPUStateActive(pVCpu))
6474	CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6475
6476	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6477
6478	#if HC_ARCH_BITS == 64
6479	/* Restore host-state bits that VT-x only restores partially. */
6480	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6481	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6482	{
6483	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6484	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6485	}
6486	#endif
6487	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6488	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6489	{
6490	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6491	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6492	}
6493
6494	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6495	VMMR0ThreadCtxHooksDeregister(pVCpu);
6496
6497	HMR0LeaveCpu(pVCpu);
6498	HM_RESTORE_PREEMPT_IF_NEEDED();
6499	return VINF_SUCCESS;
6500	}
6501
6502	Assert(pVCpu);
6503	Assert(pvUser);
6504	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6505	HMVMX_ASSERT_PREEMPT_SAFE();
6506
6507	VMMRZCallRing3Disable(pVCpu);
6508	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6509
6510	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6511	enmOperation));
6512
6513	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6514	AssertRCReturn(rc, rc);
6515
6516	VMMRZCallRing3Enable(pVCpu);
6517	return VINF_SUCCESS;
6518	}
6519
6520
6521	/**
6522	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6523	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6524	*
6525	* @param pVCpu Pointer to the VMCPU.
6526	*/
6527	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6528	{
6529	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6530	{
6531	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6532	{
6533	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6534	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6535	AssertRC(rc);
6536	}
6537	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6538	}
6539
6540
6541	/**
6542	* Evaluates the event to be delivered to the guest and sets it as the pending
6543	* event.
6544	*
6545	* @param pVCpu Pointer to the VMCPU.
6546	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6547	* out-of-sync. Make sure to update the required fields
6548	* before using them.
6549	*/
6550	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6551	{
6552	Assert(!pVCpu->hm.s.Event.fPending);
6553
6554	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6555	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6556	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6557	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6558
6559	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6560	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6561	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6562	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6563	Assert(!TRPMHasTrap(pVCpu));
6564
6565	/** @todo SMI. SMIs take priority over NMIs. */
6566	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6567	{
6568	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6569	if ( !fBlockMovSS
6570	&& !fBlockSti)
6571	{
6572	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6573	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6574	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6575	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6576
6577	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6578	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6579	}
6580	else
6581	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6582	}
6583	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6584	&& !pVCpu->hm.s.fSingleInstruction)
6585	{
6586	/*
6587	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6588	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6589	* evaluated here and not set as pending, solely based on the force-flags.
6590	*/
6591	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6592	AssertRC(rc);
6593	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6594	if ( !fBlockInt
6595	&& !fBlockSti
6596	&& !fBlockMovSS)
6597	{
6598	uint8_t u8Interrupt;
6599	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6600	if (RT_SUCCESS(rc))
6601	{
6602	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6603	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6604	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6605
6606	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6607	}
6608	else
6609	{
6610	/** @todo Does this actually happen? If not turn it into an assertion. */
6611	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6612	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6613	}
6614	}
6615	else
6616	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6617	}
6618	}
6619
6620
6621	/**
6622	* Injects any pending events into the guest if the guest is in a state to
6623	* receive them.
6624	*
6625	* @returns VBox status code (informational status codes included).
6626	* @param pVCpu Pointer to the VMCPU.
6627	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6628	* out-of-sync. Make sure to update the required fields
6629	* before using them.
6630	*/
6631	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6632	{
6633	HMVMX_ASSERT_PREEMPT_SAFE();
6634	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6635
6636	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6637	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6638	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6639	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6640
6641	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6642	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6643	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6644	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6645	Assert(!TRPMHasTrap(pVCpu));
6646
6647	int rc = VINF_SUCCESS;
6648	if (pVCpu->hm.s.Event.fPending)
6649	{
6650	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
6651	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6652	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6653	{
6654	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6655	AssertRCReturn(rc, rc);
6656	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6657	Assert(!fBlockInt);
6658	Assert(!fBlockSti);
6659	Assert(!fBlockMovSS);
6660	}
6661	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6662	{
6663	Assert(!fBlockSti);
6664	Assert(!fBlockMovSS);
6665	}
6666	#endif
6667	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6668	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6669	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6670	AssertRCReturn(rc, rc);
6671
6672	/* Update the interruptibility-state as it could have been changed by
6673	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6674	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6675	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6676
6677	#ifdef VBOX_WITH_STATISTICS
6678	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6679	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6680	else
6681	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6682	#endif
6683	}
6684
6685	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6686	int rc2 = VINF_SUCCESS;
6687	if ( fBlockSti
6688	\|\| fBlockMovSS)
6689	{
6690	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6691	{
6692	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6693	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6694	{
6695	/*
6696	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6697	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6698	* See Intel spec. 27.3.4 "Saving Non-Register State".
6699	*/
6700	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6701	AssertRCReturn(rc, rc);
6702	}
6703	}
6704	else
6705	{
6706	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6707	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6708	uIntrState = 0;
6709	}
6710	}
6711
6712	/*
6713	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6714	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6715	*/
6716	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6717	AssertRC(rc2);
6718
6719	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6720	return rc;
6721	}
6722
6723
6724	/**
6725	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6726	*
6727	* @param pVCpu Pointer to the VMCPU.
6728	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6729	* out-of-sync. Make sure to update the required fields
6730	* before using them.
6731	*/
6732	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6733	{
6734	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6735	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6736	}
6737
6738
6739	/**
6740	* Injects a double-fault (#DF) exception into the VM.
6741	*
6742	* @returns VBox status code (informational status code included).
6743	* @param pVCpu Pointer to the VMCPU.
6744	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6745	* out-of-sync. Make sure to update the required fields
6746	* before using them.
6747	*/
6748	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6749	{
6750	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6751	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6752	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6753	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6754	puIntrState);
6755	}
6756
6757
6758	/**
6759	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6760	*
6761	* @param pVCpu Pointer to the VMCPU.
6762	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6763	* out-of-sync. Make sure to update the required fields
6764	* before using them.
6765	*/
6766	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6767	{
6768	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6769	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6770	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6771	}
6772
6773
6774	/**
6775	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6776	*
6777	* @param pVCpu Pointer to the VMCPU.
6778	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6779	* out-of-sync. Make sure to update the required fields
6780	* before using them.
6781	* @param cbInstr The value of RIP that is to be pushed on the guest
6782	* stack.
6783	*/
6784	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6785	{
6786	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6787	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6788	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6789	}
6790
6791
6792	/**
6793	* Injects a general-protection (#GP) fault into the VM.
6794	*
6795	* @returns VBox status code (informational status code included).
6796	* @param pVCpu Pointer to the VMCPU.
6797	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6798	* out-of-sync. Make sure to update the required fields
6799	* before using them.
6800	* @param u32ErrorCode The error code associated with the #GP.
6801	*/
6802	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6803	uint32_t *puIntrState)
6804	{
6805	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6806	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6807	if (fErrorCodeValid)
6808	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6809	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6810	puIntrState);
6811	}
6812
6813
6814	/**
6815	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6816	*
6817	* @param pVCpu Pointer to the VMCPU.
6818	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6819	* out-of-sync. Make sure to update the required fields
6820	* before using them.
6821	* @param uVector The software interrupt vector number.
6822	* @param cbInstr The value of RIP that is to be pushed on the guest
6823	* stack.
6824	*/
6825	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6826	{
6827	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6828	if ( uVector == X86_XCPT_BP
6829	\|\| uVector == X86_XCPT_OF)
6830	{
6831	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6832	}
6833	else
6834	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6835	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6836	}
6837
6838
6839	/**
6840	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6841	* stack.
6842	*
6843	* @returns VBox status code (information status code included).
6844	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6845	* @param pVM Pointer to the VM.
6846	* @param pMixedCtx Pointer to the guest-CPU context.
6847	* @param uValue The value to push to the guest stack.
6848	*/
6849	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6850	{
6851	/*
6852	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6853	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6854	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6855	*/
6856	if (pMixedCtx->sp == 1)
6857	return VINF_EM_RESET;
6858	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6859	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6860	AssertRCReturn(rc, rc);
6861	return rc;
6862	}
6863
6864
6865	/**
6866	* Injects an event into the guest upon VM-entry by updating the relevant fields
6867	* in the VM-entry area in the VMCS.
6868	*
6869	* @returns VBox status code (informational error codes included).
6870	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6871	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6872	*
6873	* @param pVCpu Pointer to the VMCPU.
6874	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6875	* be out-of-sync. Make sure to update the required
6876	* fields before using them.
6877	* @param u64IntInfo The VM-entry interruption-information field.
6878	* @param cbInstr The VM-entry instruction length in bytes (for
6879	* software interrupts, exceptions and privileged
6880	* software exceptions).
6881	* @param u32ErrCode The VM-entry exception error code.
6882	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6883	* @param puIntrState Pointer to the current guest interruptibility-state.
6884	* This interruptibility-state will be updated if
6885	* necessary. This cannot not be NULL.
6886	*
6887	* @remarks Requires CR0!
6888	* @remarks No-long-jump zone!!!
6889	*/
6890	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
6891	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6892	{
6893	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6894	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
6895	Assert(puIntrState);
6896	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
6897
6898	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
6899	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
6900
6901	#ifdef VBOX_STRICT
6902	/* Validate the error-code-valid bit for hardware exceptions. */
6903	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6904	{
6905	switch (uVector)
6906	{
6907	case X86_XCPT_PF:
6908	case X86_XCPT_DF:
6909	case X86_XCPT_TS:
6910	case X86_XCPT_NP:
6911	case X86_XCPT_SS:
6912	case X86_XCPT_GP:
6913	case X86_XCPT_AC:
6914	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
6915	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6916	/* fallthru */
6917	default:
6918	break;
6919	}
6920	}
6921	#endif
6922
6923	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6924	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6925	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6926
6927	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6928
6929	/* We require CR0 to check if the guest is in real-mode. */
6930	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6931	AssertRCReturn(rc, rc);
6932
6933	/*
6934	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6935	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6936	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6937	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6938	*/
6939	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6940	{
6941	PVM pVM = pVCpu->CTX_SUFF(pVM);
6942	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6943	{
6944	Assert(PDMVmmDevHeapIsEnabled(pVM));
6945	Assert(pVM->hm.s.vmx.pRealModeTSS);
6946
6947	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6948	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6949	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6950	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6951	AssertRCReturn(rc, rc);
6952	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6953
6954	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6955	const size_t cbIdtEntry = sizeof(X86IDTR16);
6956	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6957	{
6958	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6959	if (uVector == X86_XCPT_DF)
6960	return VINF_EM_RESET;
6961	else if (uVector == X86_XCPT_GP)
6962	{
6963	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6964	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6965	}
6966
6967	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6968	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6969	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6970	}
6971
6972	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6973	uint16_t uGuestIp = pMixedCtx->ip;
6974	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6975	{
6976	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6977	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6978	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6979	}
6980	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6981	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6982
6983	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6984	X86IDTR16 IdtEntry;
6985	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6986	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6987	AssertRCReturn(rc, rc);
6988
6989	/* Construct the stack frame for the interrupt/exception handler. */
6990	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6991	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6992	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6993	AssertRCReturn(rc, rc);
6994
6995	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6996	if (rc == VINF_SUCCESS)
6997	{
6998	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6999	pMixedCtx->rip = IdtEntry.offSel;
7000	pMixedCtx->cs.Sel = IdtEntry.uSel;
7001	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7002	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7003	&& uVector == X86_XCPT_PF)
7004	{
7005	pMixedCtx->cr2 = GCPtrFaultAddress;
7006	}
7007
7008	/* If any other guest-state bits are changed here, make sure to update
7009	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7010	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7011	\| HM_CHANGED_GUEST_RIP
7012	\| HM_CHANGED_GUEST_RFLAGS
7013	\| HM_CHANGED_GUEST_RSP);
7014
7015	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7016	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7017	{
7018	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7019	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7020	Log4(("Clearing inhibition due to STI.\n"));
7021	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7022	}
7023	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7024
7025	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7026	it, if we are returning to ring-3 before executing guest code. */
7027	pVCpu->hm.s.Event.fPending = false;
7028	}
7029	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7030	return rc;
7031	}
7032	else
7033	{
7034	/*
7035	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7036	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7037	*/
7038	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7039	}
7040	}
7041
7042	/* Validate. */
7043	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7044	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7045	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7046
7047	/* Inject. */
7048	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7049	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7050	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7051	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7052
7053	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7054	&& uVector == X86_XCPT_PF)
7055	{
7056	pMixedCtx->cr2 = GCPtrFaultAddress;
7057	}
7058
7059	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7060	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7061
7062	AssertRCReturn(rc, rc);
7063	return rc;
7064	}
7065
7066
7067	/**
7068	* Clears the interrupt-window exiting control in the VMCS and if necessary
7069	* clears the current event in the VMCS as well.
7070	*
7071	* @returns VBox status code.
7072	* @param pVCpu Pointer to the VMCPU.
7073	*
7074	* @remarks Use this function only to clear events that have not yet been
7075	* delivered to the guest but are injected in the VMCS!
7076	* @remarks No-long-jump zone!!!
7077	*/
7078	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7079	{
7080	int rc;
7081	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7082
7083	/* Clear interrupt-window exiting control. */
7084	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7085	{
7086	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7087	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7088	AssertRC(rc);
7089	}
7090
7091	if (!pVCpu->hm.s.Event.fPending)
7092	return;
7093
7094	#ifdef VBOX_STRICT
7095	uint32_t u32EntryInfo;
7096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7097	AssertRC(rc);
7098	Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
7099	#endif
7100
7101	/* Clear the entry-interruption field (including the valid bit). */
7102	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7103	AssertRC(rc);
7104
7105	/* Clear the pending debug exception field. */
7106	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7107	AssertRC(rc);
7108	}
7109
7110
7111	/**
7112	* Enters the VT-x session.
7113	*
7114	* @returns VBox status code.
7115	* @param pVM Pointer to the VM.
7116	* @param pVCpu Pointer to the VMCPU.
7117	* @param pCpu Pointer to the CPU info struct.
7118	*/
7119	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7120	{
7121	AssertPtr(pVM);
7122	AssertPtr(pVCpu);
7123	Assert(pVM->hm.s.vmx.fSupported);
7124	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7125	NOREF(pCpu);
7126
7127	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7128	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7129
7130	#ifdef VBOX_STRICT
7131	/* Make sure we're in VMX root mode. */
7132	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7133	if (!(u32HostCR4 & X86_CR4_VMXE))
7134	{
7135	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7136	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7137	}
7138	#endif
7139
7140	/*
7141	* Load the VCPU's VMCS as the current (and active) one.
7142	*/
7143	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7144	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7145	if (RT_FAILURE(rc))
7146	return rc;
7147
7148	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7149	pVCpu->hm.s.fLeaveDone = false;
7150	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7151
7152	return VINF_SUCCESS;
7153	}
7154
7155
7156	/**
7157	* The thread-context callback (only on platforms which support it).
7158	*
7159	* @param enmEvent The thread-context event.
7160	* @param pVCpu Pointer to the VMCPU.
7161	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7162	* @thread EMT(pVCpu)
7163	*/
7164	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7165	{
7166	switch (enmEvent)
7167	{
7168	case RTTHREADCTXEVENT_PREEMPTING:
7169	{
7170	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7171	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7172	VMCPU_ASSERT_EMT(pVCpu);
7173
7174	PVM pVM = pVCpu->CTX_SUFF(pVM);
7175	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7176
7177	/* No longjmps (logger flushes, locks) in this fragile context. */
7178	VMMRZCallRing3Disable(pVCpu);
7179	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7180
7181	/*
7182	* Restore host-state (FPU, debug etc.)
7183	*/
7184	if (!pVCpu->hm.s.fLeaveDone)
7185	{
7186	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7187	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7188	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7189	pVCpu->hm.s.fLeaveDone = true;
7190	}
7191
7192	/* Leave HM context, takes care of local init (term). */
7193	int rc = HMR0LeaveCpu(pVCpu);
7194	AssertRC(rc); NOREF(rc);
7195
7196	/* Restore longjmp state. */
7197	VMMRZCallRing3Enable(pVCpu);
7198	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7199	break;
7200	}
7201
7202	case RTTHREADCTXEVENT_RESUMED:
7203	{
7204	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7205	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7206	VMCPU_ASSERT_EMT(pVCpu);
7207
7208	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7209	VMMRZCallRing3Disable(pVCpu);
7210	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7211
7212	/* Initialize the bare minimum state required for HM. This takes care of
7213	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7214	int rc = HMR0EnterCpu(pVCpu);
7215	AssertRC(rc);
7216	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7217
7218	/* Load the active VMCS as the current one. */
7219	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7220	{
7221	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7222	AssertRC(rc); NOREF(rc);
7223	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7224	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7225	}
7226	pVCpu->hm.s.fLeaveDone = false;
7227
7228	/* Restore longjmp state. */
7229	VMMRZCallRing3Enable(pVCpu);
7230	break;
7231	}
7232
7233	default:
7234	break;
7235	}
7236	}
7237
7238
7239	/**
7240	* Saves the host state in the VMCS host-state.
7241	* Sets up the VM-exit MSR-load area.
7242	*
7243	* The CPU state will be loaded from these fields on every successful VM-exit.
7244	*
7245	* @returns VBox status code.
7246	* @param pVM Pointer to the VM.
7247	* @param pVCpu Pointer to the VMCPU.
7248	*
7249	* @remarks No-long-jump zone!!!
7250	*/
7251	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7252	{
7253	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7254
7255	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7256	return VINF_SUCCESS;
7257
7258	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7259	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7260
7261	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7262	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7263
7264	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7265	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7266
7267	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7268	return rc;
7269	}
7270
7271
7272	/**
7273	* Saves the host state in the VMCS host-state.
7274	*
7275	* @returns VBox status code.
7276	* @param pVM Pointer to the VM.
7277	* @param pVCpu Pointer to the VMCPU.
7278	*
7279	* @remarks No-long-jump zone!!!
7280	*/
7281	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7282	{
7283	AssertPtr(pVM);
7284	AssertPtr(pVCpu);
7285
7286	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7287
7288	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7289	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7290	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7291	return hmR0VmxSaveHostState(pVM, pVCpu);
7292	}
7293
7294
7295	/**
7296	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7297	* loaded from these fields on every successful VM-entry.
7298	*
7299	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7300	* Sets up the VM-entry controls.
7301	* Sets up the appropriate VMX non-root function to execute guest code based on
7302	* the guest CPU mode.
7303	*
7304	* @returns VBox status code.
7305	* @param pVM Pointer to the VM.
7306	* @param pVCpu Pointer to the VMCPU.
7307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7308	* out-of-sync. Make sure to update the required fields
7309	* before using them.
7310	*
7311	* @remarks No-long-jump zone!!!
7312	*/
7313	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7314	{
7315	AssertPtr(pVM);
7316	AssertPtr(pVCpu);
7317	AssertPtr(pMixedCtx);
7318	HMVMX_ASSERT_PREEMPT_SAFE();
7319
7320	#ifdef LOG_ENABLED
7321	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7322	* probably not initialized yet? Anyway this will do for now.
7323	*
7324	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7325	* interface and disable ring-3 calls when thread-context hooks are not
7326	* available. */
7327	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7328	VMMR0LogFlushDisable(pVCpu);
7329	#endif
7330
7331	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7332
7333	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7334
7335	/* Determine real-on-v86 mode. */
7336	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7337	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7338	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7339	{
7340	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7341	}
7342
7343	/*
7344	* Load the guest-state into the VMCS.
7345	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7346	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7347	*/
7348	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7349	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7350
7351	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7352	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7353	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7354
7355	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7356	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7357	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7358
7359	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7360	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7361
7362	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7363	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7364
7365	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7366	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7367	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7368
7369	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7370	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7371
7372	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7373	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7374
7375	/*
7376	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7377	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7378	*/
7379	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7380	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7381
7382	/* Clear any unused and reserved bits. */
7383	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7384
7385	#ifdef LOG_ENABLED
7386	/* Only reenable log-flushing if the caller has it enabled. */
7387	if (!fCallerDisabledLogFlush)
7388	VMMR0LogFlushEnable(pVCpu);
7389	#endif
7390
7391	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7392	return rc;
7393	}
7394
7395
7396	/**
7397	* Loads the state shared between the host and guest into the VMCS.
7398	*
7399	* @param pVM Pointer to the VM.
7400	* @param pVCpu Pointer to the VMCPU.
7401	* @param pCtx Pointer to the guest-CPU context.
7402	*
7403	* @remarks No-long-jump zone!!!
7404	*/
7405	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7406	{
7407	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7408	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7409
7410	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7411	{
7412	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7413	AssertRC(rc);
7414	}
7415
7416	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7417	{
7418	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7419	AssertRC(rc);
7420
7421	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7422	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7423	{
7424	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7425	AssertRC(rc);
7426	}
7427	}
7428
7429	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7430	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7431	}
7432
7433
7434	/**
7435	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7436	*
7437	* @param pVM Pointer to the VM.
7438	* @param pVCpu Pointer to the VMCPU.
7439	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7440	* out-of-sync. Make sure to update the required fields
7441	* before using them.
7442	*/
7443	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7444	{
7445	HMVMX_ASSERT_PREEMPT_SAFE();
7446
7447	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7448	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7449	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7450	#endif
7451
7452	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7453	{
7454	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7455	AssertRC(rc);
7456	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7457	}
7458	else if (VMCPU_HMCF_VALUE(pVCpu))
7459	{
7460	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7461	AssertRC(rc);
7462	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7463	}
7464
7465	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7466	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7467	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7468	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7469
7470	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7471	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7472	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7473	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7474	#endif
7475	}
7476
7477
7478	/**
7479	* Does the preparations before executing guest code in VT-x.
7480	*
7481	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7482	* recompiler. We must be cautious what we do here regarding committing
7483	* guest-state information into the VMCS assuming we assuredly execute the
7484	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7485	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7486	* so that the recompiler can (and should) use them when it resumes guest
7487	* execution. Otherwise such operations must be done when we can no longer
7488	* exit to ring-3.
7489	*
7490	* @returns Strict VBox status code.
7491	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7492	* have been disabled.
7493	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7494	* double-fault into the guest.
7495	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7496	*
7497	* @param pVM Pointer to the VM.
7498	* @param pVCpu Pointer to the VMCPU.
7499	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7500	* out-of-sync. Make sure to update the required fields
7501	* before using them.
7502	* @param pVmxTransient Pointer to the VMX transient structure.
7503	*
7504	* @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7505	* interrupts will be disabled.
7506	*/
7507	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7508	{
7509	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7510
7511	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7512	PGMRZDynMapFlushAutoSet(pVCpu);
7513	#endif
7514
7515	/* Check force flag actions that might require us to go back to ring-3. */
7516	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7517	if (rc != VINF_SUCCESS)
7518	return rc;
7519
7520	#ifndef IEM_VERIFICATION_MODE_FULL
7521	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7522	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7523	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7524	{
7525	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7526	RTGCPHYS GCPhysApicBase;
7527	GCPhysApicBase = pMixedCtx->msrApicBase;
7528	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7529
7530	/* Unalias any existing mapping. */
7531	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7532	AssertRCReturn(rc, rc);
7533
7534	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7535	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7536	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7537	AssertRCReturn(rc, rc);
7538
7539	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7540	}
7541	#endif /* !IEM_VERIFICATION_MODE_FULL */
7542
7543	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7544	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7545
7546	/*
7547	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7548	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7549	*/
7550	if (TRPMHasTrap(pVCpu))
7551	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7552	else if (!pVCpu->hm.s.Event.fPending)
7553	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7554
7555	/*
7556	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7557	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7558	*/
7559	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7560	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7561	{
7562	Assert(rc == VINF_EM_RESET);
7563	return rc;
7564	}
7565
7566	/*
7567	* No longjmps to ring-3 from this point on!!!
7568	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7569	* This also disables flushing of the R0-logger instance (if any).
7570	*/
7571	VMMRZCallRing3Disable(pVCpu);
7572
7573	/*
7574	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7575	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7576	*
7577	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7578	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7579	*
7580	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7581	* executing guest code.
7582	*/
7583	pVmxTransient->uEflags = ASMIntDisableFlags();
7584	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7585	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7586	{
7587	hmR0VmxClearEventVmcs(pVCpu);
7588	ASMSetFlags(pVmxTransient->uEflags);
7589	VMMRZCallRing3Enable(pVCpu);
7590	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7591	return VINF_EM_RAW_TO_R3;
7592	}
7593	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7594	{
7595	hmR0VmxClearEventVmcs(pVCpu);
7596	ASMSetFlags(pVmxTransient->uEflags);
7597	VMMRZCallRing3Enable(pVCpu);
7598	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7599	return VINF_EM_RAW_INTERRUPT;
7600	}
7601
7602	/* We've injected any pending events. This is really the point of no return (to ring-3). */
7603	pVCpu->hm.s.Event.fPending = false;
7604
7605	return VINF_SUCCESS;
7606	}
7607
7608
7609	/**
7610	* Prepares to run guest code in VT-x and we've committed to doing so. This
7611	* means there is no backing out to ring-3 or anywhere else at this
7612	* point.
7613	*
7614	* @param pVM Pointer to the VM.
7615	* @param pVCpu Pointer to the VMCPU.
7616	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7617	* out-of-sync. Make sure to update the required fields
7618	* before using them.
7619	* @param pVmxTransient Pointer to the VMX transient structure.
7620	*
7621	* @remarks Called with preemption disabled.
7622	* @remarks No-long-jump zone!!!
7623	*/
7624	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7625	{
7626	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7627	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7628	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7629
7630	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7631	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7632
7633	/*
7634	* If we are injecting events to a real-on-v86 mode guest, we may have to update
7635	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7636	* Reload only the necessary state, the assertion will catch if other parts of the code
7637	* change.
7638	*/
7639	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7640	{
7641	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7642	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7643	}
7644
7645	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7646	if (!CPUMIsGuestFPUStateActive(pVCpu))
7647	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7648	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7649	#endif
7650
7651	/*
7652	* Load the host state bits as we may've been preempted (only happens when
7653	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7654	*/
7655	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7656	{
7657	/* This ASSUMES that pfnStartVM has been set up already. */
7658	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7659	AssertRC(rc);
7660	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7661	}
7662	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7663
7664	/*
7665	* Load the state shared between host and guest (FPU, debug).
7666	*/
7667	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7668	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7669	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7670
7671	/* Store status of the shared guest-host state at the time of VM-entry. */
7672	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7673	if (CPUMIsGuestInLongModeEx(pMixedCtx))
7674	{
7675	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7676	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7677	}
7678	else
7679	#endif
7680	{
7681	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7682	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7683	}
7684	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7685
7686	/*
7687	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7688	*/
7689	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7690	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7691
7692	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7693	RTCPUID idCurrentCpu = pCpu->idCpu;
7694	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7695	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
7696	{
7697	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7698	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7699	}
7700
7701	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7702	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7703	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7704	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7705
7706	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7707
7708	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7709	to start executing. */
7710
7711	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7712	/*
7713	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7714	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7715	*/
7716	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7717	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7718	{
7719	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7720	uint64_t u64HostTscAux = 0;
7721	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7722	AssertRC(rc2);
7723	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7724	}
7725	#endif
7726	}
7727
7728
7729	/**
7730	* Performs some essential restoration of state after running guest code in
7731	* VT-x.
7732	*
7733	* @param pVM Pointer to the VM.
7734	* @param pVCpu Pointer to the VMCPU.
7735	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7736	* out-of-sync. Make sure to update the required fields
7737	* before using them.
7738	* @param pVmxTransient Pointer to the VMX transient structure.
7739	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7740	*
7741	* @remarks Called with interrupts disabled, and returns with interrups enabled!
7742	*
7743	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7744	* unconditionally when it is safe to do so.
7745	*/
7746	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7747	{
7748	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7749
7750	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7751	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7752	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7753	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7754	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7755
7756	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7757	{
7758	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7759	/* Restore host's TSC_AUX. */
7760	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7761	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7762	#endif
7763	/** @todo Find a way to fix hardcoding a guestimate. */
7764	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7765	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7766	}
7767
7768	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7769	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7770	Assert(!(ASMGetFlags() & X86_EFL_IF));
7771	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7772
7773	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7774	if (CPUMIsGuestFPUStateActive(pVCpu))
7775	{
7776	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7777	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7778	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7779	}
7780	#endif
7781
7782	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7783	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7784	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7785	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7786
7787	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7788	uint32_t uExitReason;
7789	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7790	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7791	AssertRC(rc);
7792	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7793	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntInfo);
7794
7795	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7796	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7797	{
7798	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7799	pVmxTransient->fVMEntryFailed));
7800	return;
7801	}
7802
7803	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7804	{
7805	/* Update the guest interruptibility-state from the VMCS. */
7806	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7807	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7808	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7809	AssertRC(rc);
7810	#endif
7811	/*
7812	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7813	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7814	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7815	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7816	*/
7817	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7818	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7819	{
7820	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7821	AssertRC(rc);
7822	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
7823	}
7824	}
7825	}
7826
7827
7828
7829	/**
7830	* Runs the guest code using VT-x the normal way.
7831	*
7832	* @returns VBox status code.
7833	* @param pVM Pointer to the VM.
7834	* @param pVCpu Pointer to the VMCPU.
7835	* @param pCtx Pointer to the guest-CPU context.
7836	*
7837	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
7838	* @remarks Called with preemption disabled.
7839	*/
7840	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7841	{
7842	VMXTRANSIENT VmxTransient;
7843	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7844	int rc = VERR_INTERNAL_ERROR_5;
7845	uint32_t cLoops = 0;
7846
7847	for (;; cLoops++)
7848	{
7849	Assert(!HMR0SuspendPending());
7850	HMVMX_ASSERT_CPU_SAFE();
7851
7852	/* Preparatory work for running guest code, this may force us to return
7853	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7854	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7855	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7856	if (rc != VINF_SUCCESS)
7857	break;
7858
7859	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7860	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7861	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7862
7863	/* Restore any residual host-state and save any bits shared between host
7864	and guest into the guest-CPU state. Re-enables interrupts! */
7865	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7866
7867	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7868	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7869	{
7870	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7871	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7872	return rc;
7873	}
7874
7875	/* Handle the VM-exit. */
7876	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7877	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7878	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7879	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7880	HMVMX_START_EXIT_DISPATCH_PROF();
7881	#ifdef HMVMX_USE_FUNCTION_TABLE
7882	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7883	#else
7884	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7885	#endif
7886	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7887	if (rc != VINF_SUCCESS)
7888	break;
7889	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7890	{
7891	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7892	rc = VINF_EM_RAW_INTERRUPT;
7893	break;
7894	}
7895	}
7896
7897	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7898	return rc;
7899	}
7900
7901
7902	/**
7903	* Single steps guest code using VT-x.
7904	*
7905	* @returns VBox status code.
7906	* @param pVM Pointer to the VM.
7907	* @param pVCpu Pointer to the VMCPU.
7908	* @param pCtx Pointer to the guest-CPU context.
7909	*
7910	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7911	* @remarks Called with preemption disabled.
7912	*/
7913	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7914	{
7915	VMXTRANSIENT VmxTransient;
7916	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7917	int rc = VERR_INTERNAL_ERROR_5;
7918	uint32_t cLoops = 0;
7919	uint16_t uCsStart = pCtx->cs.Sel;
7920	uint64_t uRipStart = pCtx->rip;
7921
7922	for (;; cLoops++)
7923	{
7924	Assert(!HMR0SuspendPending());
7925	HMVMX_ASSERT_CPU_SAFE();
7926
7927	/* Preparatory work for running guest code, this may force us to return
7928	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7929	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7930	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7931	if (rc != VINF_SUCCESS)
7932	break;
7933
7934	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7935	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7936	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7937
7938	/* Restore any residual host-state and save any bits shared between host
7939	and guest into the guest-CPU state. Re-enables interrupts! */
7940	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7941
7942	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7943	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7944	{
7945	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7946	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7947	return rc;
7948	}
7949
7950	/* Handle the VM-exit. */
7951	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7952	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7953	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7954	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7955	HMVMX_START_EXIT_DISPATCH_PROF();
7956	#ifdef HMVMX_USE_FUNCTION_TABLE
7957	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7958	#else
7959	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7960	#endif
7961	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7962	if (rc != VINF_SUCCESS)
7963	break;
7964	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7965	{
7966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7967	rc = VINF_EM_RAW_INTERRUPT;
7968	break;
7969	}
7970
7971	/*
7972	* Did the RIP change, if so, consider it a single step.
7973	* Otherwise, make sure one of the TFs gets set.
7974	*/
7975	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7976	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7977	AssertRCReturn(rc2, rc2);
7978	if ( pCtx->rip != uRipStart
7979	\|\| pCtx->cs.Sel != uCsStart)
7980	{
7981	rc = VINF_EM_DBG_STEPPED;
7982	break;
7983	}
7984	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7985	}
7986
7987	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7988	return rc;
7989	}
7990
7991
7992	/**
7993	* Runs the guest code using VT-x.
7994	*
7995	* @returns VBox status code.
7996	* @param pVM Pointer to the VM.
7997	* @param pVCpu Pointer to the VMCPU.
7998	* @param pCtx Pointer to the guest-CPU context.
7999	*
8000	* @remarks Called with preemption disabled.
8001	*/
8002	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8003	{
8004	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8005	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8006	HMVMX_ASSERT_PREEMPT_SAFE();
8007
8008	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8009
8010	int rc;
8011	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8012	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8013	else
8014	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8015
8016	if (rc == VERR_EM_INTERPRETER)
8017	rc = VINF_EM_RAW_EMULATE_INSTR;
8018	else if (rc == VINF_EM_RESET)
8019	rc = VINF_EM_TRIPLE_FAULT;
8020
8021	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8022	if (RT_FAILURE(rc2))
8023	{
8024	pVCpu->hm.s.u32HMError = rc;
8025	rc = rc2;
8026	}
8027	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8028	return rc;
8029	}
8030
8031
8032	#ifndef HMVMX_USE_FUNCTION_TABLE
8033	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8034	{
8035	int rc;
8036	switch (rcReason)
8037	{
8038	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8039	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8040	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8041	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8042	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8043	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8044	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8045	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8046	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8047	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8048	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8049	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8050	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8051	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8052	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8053	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8054	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8055	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8056	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8057	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8058	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8059	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8060	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8061	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8062	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8063	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8064	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8065	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8066	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8067	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8068	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8069	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8070	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8071
8072	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8073	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8074	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8075	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8076	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8077	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8078	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8079	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8080	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8081
8082	case VMX_EXIT_VMCALL:
8083	case VMX_EXIT_VMCLEAR:
8084	case VMX_EXIT_VMLAUNCH:
8085	case VMX_EXIT_VMPTRLD:
8086	case VMX_EXIT_VMPTRST:
8087	case VMX_EXIT_VMREAD:
8088	case VMX_EXIT_VMRESUME:
8089	case VMX_EXIT_VMWRITE:
8090	case VMX_EXIT_VMXOFF:
8091	case VMX_EXIT_VMXON:
8092	case VMX_EXIT_INVEPT:
8093	case VMX_EXIT_INVVPID:
8094	case VMX_EXIT_VMFUNC:
8095	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8096	break;
8097	default:
8098	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8099	break;
8100	}
8101	return rc;
8102	}
8103	#endif
8104
8105	#ifdef DEBUG
8106	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8107	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8108	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8109
8110	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8111	do \
8112	{ \
8113	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8114	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8115	} while (0)
8116
8117	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8118	do { \
8119	AssertPtr(pVCpu); \
8120	AssertPtr(pMixedCtx); \
8121	AssertPtr(pVmxTransient); \
8122	Assert(pVmxTransient->fVMEntryFailed == false); \
8123	Assert(ASMIntAreEnabled()); \
8124	HMVMX_ASSERT_PREEMPT_SAFE(); \
8125	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8126	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8127	HMVMX_ASSERT_PREEMPT_SAFE(); \
8128	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8129	HMVMX_ASSERT_PREEMPT_CPUID(); \
8130	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8131	} while (0)
8132
8133	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8134	do { \
8135	Log4Func(("\n")); \
8136	} while(0)
8137	#else /* Release builds */
8138	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8139	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8140	#endif
8141
8142
8143	/**
8144	* Advances the guest RIP after reading it from the VMCS.
8145	*
8146	* @returns VBox status code.
8147	* @param pVCpu Pointer to the VMCPU.
8148	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8149	* out-of-sync. Make sure to update the required fields
8150	* before using them.
8151	* @param pVmxTransient Pointer to the VMX transient structure.
8152	*
8153	* @remarks No-long-jump zone!!!
8154	*/
8155	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8156	{
8157	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8158	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8159	AssertRCReturn(rc, rc);
8160
8161	pMixedCtx->rip += pVmxTransient->cbInstr;
8162	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8163	return rc;
8164	}
8165
8166
8167	/**
8168	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8169	* and update error record fields accordingly.
8170	*
8171	* @return VMX_IGS_* return codes.
8172	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8173	* wrong with the guest state.
8174	*
8175	* @param pVM Pointer to the VM.
8176	* @param pVCpu Pointer to the VMCPU.
8177	* @param pCtx Pointer to the guest-CPU state.
8178	*/
8179	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8180	{
8181	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8182	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8183	uError = (err); \
8184	break; \
8185	} else do {} while (0)
8186	/* Duplicate of IEM_IS_CANONICAL(). */
8187	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8188
8189	int rc;
8190	uint32_t uError = VMX_IGS_ERROR;
8191	uint32_t u32Val;
8192	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8193
8194	do
8195	{
8196	/*
8197	* CR0.
8198	*/
8199	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8200	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8201	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8202	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8203	if (fUnrestrictedGuest)
8204	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8205
8206	uint32_t u32GuestCR0;
8207	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8208	AssertRCBreak(rc);
8209	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8210	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8211	if ( !fUnrestrictedGuest
8212	&& (u32GuestCR0 & X86_CR0_PG)
8213	&& !(u32GuestCR0 & X86_CR0_PE))
8214	{
8215	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8216	}
8217
8218	/*
8219	* CR4.
8220	*/
8221	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8222	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8223
8224	uint32_t u32GuestCR4;
8225	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8226	AssertRCBreak(rc);
8227	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8228	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8229
8230	/*
8231	* IA32_DEBUGCTL MSR.
8232	*/
8233	uint64_t u64Val;
8234	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8235	AssertRCBreak(rc);
8236	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8237	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8238	{
8239	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8240	}
8241	uint64_t u64DebugCtlMsr = u64Val;
8242
8243	#ifdef VBOX_STRICT
8244	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8245	AssertRCBreak(rc);
8246	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8247	#endif
8248	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8249
8250	/*
8251	* RIP and RFLAGS.
8252	*/
8253	uint32_t u32Eflags;
8254	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8255	if (HMVMX_IS_64BIT_HOST_MODE())
8256	{
8257	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8258	AssertRCBreak(rc);
8259	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8260	if ( !fLongModeGuest
8261	\|\| !pCtx->cs.Attr.n.u1Long)
8262	{
8263	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8264	}
8265	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8266	* must be identical if the "IA32e mode guest" VM-entry control is 1
8267	* and CS.L is 1. No check applies if the CPU supports 64
8268	* linear-address bits. */
8269
8270	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8271	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8272	AssertRCBreak(rc);
8273	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8274	VMX_IGS_RFLAGS_RESERVED);
8275	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8276	u32Eflags = u64Val;
8277	}
8278	else
8279	#endif
8280	{
8281	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8282	AssertRCBreak(rc);
8283	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8284	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8285	}
8286
8287	if ( fLongModeGuest
8288	\|\| ( fUnrestrictedGuest
8289	&& !(u32GuestCR0 & X86_CR0_PE)))
8290	{
8291	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8292	}
8293
8294	uint32_t u32EntryInfo;
8295	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8296	AssertRCBreak(rc);
8297	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8298	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8299	{
8300	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8301	}
8302
8303	/*
8304	* 64-bit checks.
8305	*/
8306	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8307	if (HMVMX_IS_64BIT_HOST_MODE())
8308	{
8309	if ( fLongModeGuest
8310	&& !fUnrestrictedGuest)
8311	{
8312	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8313	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8314	}
8315
8316	if ( !fLongModeGuest
8317	&& (u32GuestCR4 & X86_CR4_PCIDE))
8318	{
8319	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8320	}
8321
8322	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8323	* 51:32 beyond the processor's physical-address width are 0. */
8324
8325	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8326	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8327	{
8328	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8329	}
8330
8331	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8332	AssertRCBreak(rc);
8333	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8334
8335	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8336	AssertRCBreak(rc);
8337	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8338	}
8339	#endif
8340
8341	/*
8342	* PERF_GLOBAL MSR.
8343	*/
8344	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8345	{
8346	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8347	AssertRCBreak(rc);
8348	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8349	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8350	}
8351
8352	/*
8353	* PAT MSR.
8354	*/
8355	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8356	{
8357	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8358	AssertRCBreak(rc);
8359	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8360	for (unsigned i = 0; i < 8; i++)
8361	{
8362	uint8_t u8Val = (u64Val & 0x7);
8363	if ( u8Val != 0 /* UC */
8364	\|\| u8Val != 1 /* WC */
8365	\|\| u8Val != 4 /* WT */
8366	\|\| u8Val != 5 /* WP */
8367	\|\| u8Val != 6 /* WB */
8368	\|\| u8Val != 7 /* UC- */)
8369	{
8370	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8371	}
8372	u64Val >>= 3;
8373	}
8374	}
8375
8376	/*
8377	* EFER MSR.
8378	*/
8379	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8380	{
8381	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8382	AssertRCBreak(rc);
8383	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8384	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8385	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8386	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8387	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8388	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8389	}
8390
8391	/*
8392	* Segment registers.
8393	*/
8394	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8395	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8396	if (!(u32Eflags & X86_EFL_VM))
8397	{
8398	/* CS */
8399	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8400	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8401	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8402	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8403	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8404	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8405	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8406	/* CS cannot be loaded with NULL in protected mode. */
8407	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8408	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8409	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8410	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8411	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8412	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8413	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8414	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8415	else
8416	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8417
8418	/* SS */
8419	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8420	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8421	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8422	if ( !(pCtx->cr0 & X86_CR0_PE)
8423	\|\| pCtx->cs.Attr.n.u4Type == 3)
8424	{
8425	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8426	}
8427	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8428	{
8429	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8430	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8431	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8432	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8433	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8434	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8435	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8436	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8437	}
8438
8439	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8440	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8441	{
8442	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8443	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8444	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8445	\|\| pCtx->ds.Attr.n.u4Type > 11
8446	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8447	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8448	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8449	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8450	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8451	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8452	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8453	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8454	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8455	}
8456	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8457	{
8458	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8459	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8460	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8461	\|\| pCtx->es.Attr.n.u4Type > 11
8462	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8463	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8464	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8465	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8466	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8467	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8468	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8469	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8470	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8471	}
8472	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8473	{
8474	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8475	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8476	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8477	\|\| pCtx->fs.Attr.n.u4Type > 11
8478	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8479	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8480	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8481	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8482	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8483	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8484	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8485	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8486	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8487	}
8488	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8489	{
8490	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8491	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8492	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8493	\|\| pCtx->gs.Attr.n.u4Type > 11
8494	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8495	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8496	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8497	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8498	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8499	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8500	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8501	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8502	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8503	}
8504	/* 64-bit capable CPUs. */
8505	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8506	if (HMVMX_IS_64BIT_HOST_MODE())
8507	{
8508	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8509	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8510	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8511	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8512	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8513	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8514	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8515	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8516	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8517	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8518	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8519	}
8520	#endif
8521	}
8522	else
8523	{
8524	/* V86 mode checks. */
8525	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8526	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8527	{
8528	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8529	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8530	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8531	}
8532	else
8533	{
8534	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8535	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8536	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8537	}
8538
8539	/* CS */
8540	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8541	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8542	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8543	/* SS */
8544	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8545	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8546	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8547	/* DS */
8548	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8549	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8550	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8551	/* ES */
8552	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8553	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8554	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8555	/* FS */
8556	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8557	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8558	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8559	/* GS */
8560	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8561	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8562	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8563	/* 64-bit capable CPUs. */
8564	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8565	if (HMVMX_IS_64BIT_HOST_MODE())
8566	{
8567	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8568	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8569	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8570	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8571	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8572	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8573	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8574	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8575	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8576	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8577	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8578	}
8579	#endif
8580	}
8581
8582	/*
8583	* TR.
8584	*/
8585	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8586	/* 64-bit capable CPUs. */
8587	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8588	if (HMVMX_IS_64BIT_HOST_MODE())
8589	{
8590	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8591	}
8592	#endif
8593	if (fLongModeGuest)
8594	{
8595	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8596	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8597	}
8598	else
8599	{
8600	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8601	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8602	VMX_IGS_TR_ATTR_TYPE_INVALID);
8603	}
8604	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8605	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8606	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8607	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8608	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8609	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8610	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8611	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8612
8613	/*
8614	* GDTR and IDTR.
8615	*/
8616	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8617	if (HMVMX_IS_64BIT_HOST_MODE())
8618	{
8619	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8620	AssertRCBreak(rc);
8621	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8622
8623	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8624	AssertRCBreak(rc);
8625	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8626	}
8627	#endif
8628
8629	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8630	AssertRCBreak(rc);
8631	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8632
8633	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8634	AssertRCBreak(rc);
8635	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8636
8637	/*
8638	* Guest Non-Register State.
8639	*/
8640	/* Activity State. */
8641	uint32_t u32ActivityState;
8642	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8643	AssertRCBreak(rc);
8644	HMVMX_CHECK_BREAK( !u32ActivityState
8645	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8646	VMX_IGS_ACTIVITY_STATE_INVALID);
8647	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8648	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8649	uint32_t u32IntrState;
8650	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8651	AssertRCBreak(rc);
8652	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8653	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8654	{
8655	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8656	}
8657
8658	/** @todo Activity state and injecting interrupts. Left as a todo since we
8659	* currently don't use activity states but ACTIVE. */
8660
8661	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8662	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8663
8664	/* Guest interruptibility-state. */
8665	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8666	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8667	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8668	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8669	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8670	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8671	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8672	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8673	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8674	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8675	{
8676	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8677	{
8678	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8679	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8680	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8681	}
8682	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8683	{
8684	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8685	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8686	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8687	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8688	}
8689	}
8690	/** @todo Assumes the processor is not in SMM. */
8691	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8692	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8693	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8694	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8695	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8696	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8697	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8698	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8699	{
8700	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8701	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8702	}
8703
8704	/* Pending debug exceptions. */
8705	if (HMVMX_IS_64BIT_HOST_MODE())
8706	{
8707	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8708	AssertRCBreak(rc);
8709	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8710	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8711	u32Val = u64Val; /* For pending debug exceptions checks below. */
8712	}
8713	else
8714	{
8715	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8716	AssertRCBreak(rc);
8717	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8718	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8719	}
8720
8721	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8722	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8723	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8724	{
8725	if ( (u32Eflags & X86_EFL_TF)
8726	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8727	{
8728	/* Bit 14 is PendingDebug.BS. */
8729	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8730	}
8731	if ( !(u32Eflags & X86_EFL_TF)
8732	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8733	{
8734	/* Bit 14 is PendingDebug.BS. */
8735	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8736	}
8737	}
8738
8739	/* VMCS link pointer. */
8740	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8741	AssertRCBreak(rc);
8742	if (u64Val != UINT64_C(0xffffffffffffffff))
8743	{
8744	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8745	/** @todo Bits beyond the processor's physical-address width MBZ. */
8746	/** @todo 32-bit located in memory referenced by value of this field (as a
8747	* physical address) must contain the processor's VMCS revision ID. */
8748	/** @todo SMM checks. */
8749	}
8750
8751	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8752
8753	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8754	if (uError == VMX_IGS_ERROR)
8755	uError = VMX_IGS_REASON_NOT_FOUND;
8756	} while (0);
8757
8758	pVCpu->hm.s.u32HMError = uError;
8759	return uError;
8760
8761	#undef HMVMX_ERROR_BREAK
8762	#undef HMVMX_CHECK_BREAK
8763	#undef HMVMX_IS_CANONICAL
8764	}
8765
8766	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8767	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8768	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8769
8770	/** @name VM-exit handlers.
8771	* @{
8772	*/
8773
8774	/**
8775	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8776	*/
8777	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8778	{
8779	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8780	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8781	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8782	#if HC_ARCH_BITS == 64
8783	Assert(ASMIntAreEnabled());
8784	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8785	return VINF_SUCCESS;
8786	#endif
8787	return VINF_EM_RAW_INTERRUPT;
8788	}
8789
8790
8791	/**
8792	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8793	*/
8794	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8795	{
8796	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8797	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8798
8799	int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
8800	AssertRCReturn(rc, rc);
8801
8802	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
8803	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8804	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8805	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
8806
8807	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8808	{
8809	/*
8810	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8811	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8812	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8813	*
8814	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8815	*/
8816	VMXDispatchHostNmi();
8817	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8818	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8819	return VINF_SUCCESS;
8820	}
8821
8822	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8823	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8824	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8825	{
8826	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8827	return VINF_SUCCESS;
8828	}
8829	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8830	{
8831	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8832	return rc;
8833	}
8834
8835	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
8836	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
8837	switch (uIntType)
8838	{
8839	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8840	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8841	/* no break */
8842	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8843	{
8844	switch (uVector)
8845	{
8846	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8847	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8848	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8849	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8850	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8851	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8852	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8853	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8854	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8855	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8856	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8857	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8858	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8859	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8860	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8861	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8862	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8863	#endif
8864	default:
8865	{
8866	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8867	AssertRCReturn(rc, rc);
8868
8869	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8870	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8871	{
8872	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8873	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8874	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8875
8876	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8877	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
8878	AssertRCReturn(rc, rc);
8879	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
8880	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
8881	0 /* GCPtrFaultAddress */);
8882	AssertRCReturn(rc, rc);
8883	}
8884	else
8885	{
8886	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8887	pVCpu->hm.s.u32HMError = uVector;
8888	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8889	}
8890	break;
8891	}
8892	}
8893	break;
8894	}
8895
8896	default:
8897	{
8898	pVCpu->hm.s.u32HMError = uExitIntInfo;
8899	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
8900	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
8901	break;
8902	}
8903	}
8904	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8905	return rc;
8906	}
8907
8908
8909	/**
8910	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8911	*/
8912	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8913	{
8914	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8915
8916	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8917	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8918	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8919	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8920	AssertRCReturn(rc, rc);
8921
8922	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8923	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8924	return VINF_SUCCESS;
8925	}
8926
8927
8928	/**
8929	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8930	*/
8931	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8932	{
8933	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8934	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8935	HMVMX_RETURN_UNEXPECTED_EXIT();
8936	}
8937
8938
8939	/**
8940	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8941	*/
8942	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8943	{
8944	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8945	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8946	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8947	}
8948
8949
8950	/**
8951	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8952	*/
8953	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8954	{
8955	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8957	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8958	}
8959
8960
8961	/**
8962	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8963	*/
8964	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8965	{
8966	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8967	PVM pVM = pVCpu->CTX_SUFF(pVM);
8968	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8969	if (RT_LIKELY(rc == VINF_SUCCESS))
8970	{
8971	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8972	Assert(pVmxTransient->cbInstr == 2);
8973	}
8974	else
8975	{
8976	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8977	rc = VERR_EM_INTERPRETER;
8978	}
8979	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8980	return rc;
8981	}
8982
8983
8984	/**
8985	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8986	*/
8987	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8988	{
8989	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8990	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8991	AssertRCReturn(rc, rc);
8992
8993	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8994	return VINF_EM_RAW_EMULATE_INSTR;
8995
8996	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8997	HMVMX_RETURN_UNEXPECTED_EXIT();
8998	}
8999
9000
9001	/**
9002	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9003	*/
9004	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9005	{
9006	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9007	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9008	AssertRCReturn(rc, rc);
9009
9010	PVM pVM = pVCpu->CTX_SUFF(pVM);
9011	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9012	if (RT_LIKELY(rc == VINF_SUCCESS))
9013	{
9014	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9015	Assert(pVmxTransient->cbInstr == 2);
9016	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9017	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9018	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9019	}
9020	else
9021	{
9022	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9023	rc = VERR_EM_INTERPRETER;
9024	}
9025	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9026	return rc;
9027	}
9028
9029
9030	/**
9031	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9032	*/
9033	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9034	{
9035	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9036	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9037	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9038	AssertRCReturn(rc, rc);
9039
9040	PVM pVM = pVCpu->CTX_SUFF(pVM);
9041	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9042	if (RT_LIKELY(rc == VINF_SUCCESS))
9043	{
9044	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9045	Assert(pVmxTransient->cbInstr == 3);
9046	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9047	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9048	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9049	}
9050	else
9051	{
9052	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9053	rc = VERR_EM_INTERPRETER;
9054	}
9055	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9056	return rc;
9057	}
9058
9059
9060	/**
9061	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9062	*/
9063	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9064	{
9065	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9066	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9067	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9068	AssertRCReturn(rc, rc);
9069
9070	PVM pVM = pVCpu->CTX_SUFF(pVM);
9071	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9072	if (RT_LIKELY(rc == VINF_SUCCESS))
9073	{
9074	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9075	Assert(pVmxTransient->cbInstr == 2);
9076	}
9077	else
9078	{
9079	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9080	rc = VERR_EM_INTERPRETER;
9081	}
9082	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9083	return rc;
9084	}
9085
9086
9087	/**
9088	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9089	*/
9090	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9091	{
9092	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9093	PVM pVM = pVCpu->CTX_SUFF(pVM);
9094	Assert(!pVM->hm.s.fNestedPaging);
9095
9096	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9097	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9098	AssertRCReturn(rc, rc);
9099
9100	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9101	rc = VBOXSTRICTRC_VAL(rc2);
9102	if (RT_LIKELY(rc == VINF_SUCCESS))
9103	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9104	else
9105	{
9106	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9107	pVmxTransient->uExitQualification, rc));
9108	}
9109	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9110	return rc;
9111	}
9112
9113
9114	/**
9115	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9116	*/
9117	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9118	{
9119	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9120	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9121	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9122	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9123	AssertRCReturn(rc, rc);
9124
9125	PVM pVM = pVCpu->CTX_SUFF(pVM);
9126	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9127	if (RT_LIKELY(rc == VINF_SUCCESS))
9128	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9129	else
9130	{
9131	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9132	rc = VERR_EM_INTERPRETER;
9133	}
9134	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9135	return rc;
9136	}
9137
9138
9139	/**
9140	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9141	*/
9142	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9143	{
9144	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9145	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9146	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9147	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9148	AssertRCReturn(rc, rc);
9149
9150	PVM pVM = pVCpu->CTX_SUFF(pVM);
9151	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9152	rc = VBOXSTRICTRC_VAL(rc2);
9153	if (RT_LIKELY( rc == VINF_SUCCESS
9154	\|\| rc == VINF_EM_HALT))
9155	{
9156	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9157	AssertRCReturn(rc3, rc3);
9158
9159	if ( rc == VINF_EM_HALT
9160	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9161	{
9162	rc = VINF_SUCCESS;
9163	}
9164	}
9165	else
9166	{
9167	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9168	rc = VERR_EM_INTERPRETER;
9169	}
9170	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9171	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9172	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9173	return rc;
9174	}
9175
9176
9177	/**
9178	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9179	*/
9180	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9181	{
9182	/*
9183	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9184	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9185	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9186	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9187	*/
9188	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9189	HMVMX_RETURN_UNEXPECTED_EXIT();
9190	}
9191
9192
9193	/**
9194	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9195	*/
9196	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9197	{
9198	/*
9199	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9200	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9201	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9202	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9203	*/
9204	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9205	HMVMX_RETURN_UNEXPECTED_EXIT();
9206	}
9207
9208
9209	/**
9210	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9211	*/
9212	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9213	{
9214	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9215	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9216	HMVMX_RETURN_UNEXPECTED_EXIT();
9217	}
9218
9219
9220	/**
9221	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9222	*/
9223	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9224	{
9225	/*
9226	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9227	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9228	* See Intel spec. 25.3 "Other Causes of VM-exits".
9229	*/
9230	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9231	HMVMX_RETURN_UNEXPECTED_EXIT();
9232	}
9233
9234
9235	/**
9236	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9237	* VM-exit.
9238	*/
9239	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9240	{
9241	/*
9242	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9243	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9244	*
9245	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9246	* See Intel spec. "23.8 Restrictions on VMX operation".
9247	*/
9248	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9249	return VINF_SUCCESS;
9250	}
9251
9252
9253	/**
9254	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9255	* VM-exit.
9256	*/
9257	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9258	{
9259	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9260	return VINF_EM_RESET;
9261	}
9262
9263
9264	/**
9265	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9266	*/
9267	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9268	{
9269	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9270	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9271	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9272	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9273	AssertRCReturn(rc, rc);
9274
9275	pMixedCtx->rip++;
9276	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9277	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9278	rc = VINF_SUCCESS;
9279	else
9280	rc = VINF_EM_HALT;
9281
9282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9283	return rc;
9284	}
9285
9286
9287	/**
9288	* VM-exit handler for instructions that result in a #UD exception delivered to
9289	* the guest.
9290	*/
9291	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9292	{
9293	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9294	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9295	return VINF_SUCCESS;
9296	}
9297
9298
9299	/**
9300	* VM-exit handler for expiry of the VMX preemption timer.
9301	*/
9302	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9303	{
9304	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9305
9306	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9307	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9308
9309	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9310	PVM pVM = pVCpu->CTX_SUFF(pVM);
9311	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9312	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9313	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9314	}
9315
9316
9317	/**
9318	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9319	*/
9320	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9321	{
9322	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9323
9324	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9325	/** @todo check if XSETBV is supported by the recompiler. */
9326	return VERR_EM_INTERPRETER;
9327	}
9328
9329
9330	/**
9331	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9332	*/
9333	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9334	{
9335	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9336
9337	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9338	/** @todo implement EMInterpretInvpcid() */
9339	return VERR_EM_INTERPRETER;
9340	}
9341
9342
9343	/**
9344	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9345	* Error VM-exit.
9346	*/
9347	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9348	{
9349	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9350	AssertRCReturn(rc, rc);
9351
9352	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9353	NOREF(uInvalidReason);
9354
9355	#ifdef VBOX_STRICT
9356	uint32_t uIntrState;
9357	HMVMXHCUINTREG uHCReg;
9358	uint64_t u64Val;
9359	uint32_t u32Val;
9360
9361	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9362	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9363	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9364	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9365	AssertRCReturn(rc, rc);
9366
9367	Log4(("uInvalidReason %u\n", uInvalidReason));
9368	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9369	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9370	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9371	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9372
9373	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9374	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9375	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9376	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9377	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9378	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9379	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9380	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9381	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9382	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9383	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9384	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9385	#endif
9386
9387	PVM pVM = pVCpu->CTX_SUFF(pVM);
9388	HMDumpRegs(pVM, pVCpu, pMixedCtx);
9389
9390	return VERR_VMX_INVALID_GUEST_STATE;
9391	}
9392
9393
9394	/**
9395	* VM-exit handler for VM-entry failure due to an MSR-load
9396	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9397	*/
9398	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9399	{
9400	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9401	HMVMX_RETURN_UNEXPECTED_EXIT();
9402	}
9403
9404
9405	/**
9406	* VM-exit handler for VM-entry failure due to a machine-check event
9407	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9408	*/
9409	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9410	{
9411	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9412	HMVMX_RETURN_UNEXPECTED_EXIT();
9413	}
9414
9415
9416	/**
9417	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9418	* theory.
9419	*/
9420	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9421	{
9422	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9423	return VERR_VMX_UNDEFINED_EXIT_CODE;
9424	}
9425
9426
9427	/**
9428	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9429	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9430	* Conditional VM-exit.
9431	*/
9432	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9433	{
9434	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9435
9436	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9437	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9438	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9439	return VERR_EM_INTERPRETER;
9440	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9441	HMVMX_RETURN_UNEXPECTED_EXIT();
9442	}
9443
9444
9445	/**
9446	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9447	*/
9448	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9449	{
9450	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9451
9452	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9453	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9454	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9455	return VERR_EM_INTERPRETER;
9456	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9457	HMVMX_RETURN_UNEXPECTED_EXIT();
9458	}
9459
9460
9461	/**
9462	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9463	*/
9464	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9465	{
9466	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9467
9468	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9469	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9470	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9471	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9472	AssertRCReturn(rc, rc);
9473	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9474
9475	PVM pVM = pVCpu->CTX_SUFF(pVM);
9476	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9477	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9478	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9480
9481	if (RT_LIKELY(rc == VINF_SUCCESS))
9482	{
9483	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9484	Assert(pVmxTransient->cbInstr == 2);
9485	}
9486	return rc;
9487	}
9488
9489
9490	/**
9491	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9492	*/
9493	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9494	{
9495	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9496	PVM pVM = pVCpu->CTX_SUFF(pVM);
9497	int rc = VINF_SUCCESS;
9498
9499	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9500	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9501	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9502	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9503	AssertRCReturn(rc, rc);
9504	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9505
9506	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9507	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9508	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9509
9510	if (RT_LIKELY(rc == VINF_SUCCESS))
9511	{
9512	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9513
9514	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9515	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9516	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9517	{
9518	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9519	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9520	EMInterpretWrmsr() changes it. */
9521	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9522	}
9523	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9524	{
9525	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9526	AssertRCReturn(rc, rc);
9527	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9528	}
9529	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9530	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9531
9532	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9533	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9534	{
9535	switch (pMixedCtx->ecx)
9536	{
9537	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9538	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9539	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9540	case MSR_K8_FS_BASE: /* no break */
9541	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9542	case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9543	}
9544	}
9545	#ifdef VBOX_STRICT
9546	else
9547	{
9548	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9549	switch (pMixedCtx->ecx)
9550	{
9551	case MSR_IA32_SYSENTER_CS:
9552	case MSR_IA32_SYSENTER_EIP:
9553	case MSR_IA32_SYSENTER_ESP:
9554	case MSR_K8_FS_BASE:
9555	case MSR_K8_GS_BASE:
9556	{
9557	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9558	HMVMX_RETURN_UNEXPECTED_EXIT();
9559	}
9560
9561	case MSR_K8_LSTAR:
9562	case MSR_K6_STAR:
9563	case MSR_K8_SF_MASK:
9564	case MSR_K8_TSC_AUX:
9565	case MSR_K8_KERNEL_GS_BASE:
9566	{
9567	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9568	pMixedCtx->ecx));
9569	HMVMX_RETURN_UNEXPECTED_EXIT();
9570	}
9571	}
9572	}
9573	#endif /* VBOX_STRICT */
9574	}
9575	return rc;
9576	}
9577
9578
9579	/**
9580	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9581	*/
9582	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9583	{
9584	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9585
9586	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9587	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9588	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9589	return VERR_EM_INTERPRETER;
9590	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9591	HMVMX_RETURN_UNEXPECTED_EXIT();
9592	}
9593
9594
9595	/**
9596	* VM-exit handler for when the TPR value is lowered below the specified
9597	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9598	*/
9599	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9600	{
9601	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9602	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9603
9604	/*
9605	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9606	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9607	* resume guest execution.
9608	*/
9609	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9611	return VINF_SUCCESS;
9612	}
9613
9614
9615	/**
9616	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9617	* VM-exit.
9618	*
9619	* @retval VINF_SUCCESS when guest execution can continue.
9620	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9621	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9622	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9623	* recompiler.
9624	*/
9625	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9626	{
9627	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9628	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9629	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9630	AssertRCReturn(rc, rc);
9631
9632	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9633	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9634	PVM pVM = pVCpu->CTX_SUFF(pVM);
9635	switch (uAccessType)
9636	{
9637	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9638	{
9639	#if 0
9640	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9641	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9642	#else
9643	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9644	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9645	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9646	#endif
9647	AssertRCReturn(rc, rc);
9648
9649	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9650	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9651	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9652	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9653
9654	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9655	{
9656	case 0: /* CR0 */
9657	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9658	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9659	break;
9660	case 2: /* CR2 */
9661	/* Nothing to do here, CR2 it's not part of the VMCS. */
9662	break;
9663	case 3: /* CR3 */
9664	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9665	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9666	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9667	break;
9668	case 4: /* CR4 */
9669	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9670	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9671	break;
9672	case 8: /* CR8 */
9673	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9674	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9675	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9676	break;
9677	default:
9678	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9679	break;
9680	}
9681
9682	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9683	break;
9684	}
9685
9686	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9687	{
9688	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9689	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9690	AssertRCReturn(rc, rc);
9691	Assert( !pVM->hm.s.fNestedPaging
9692	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9693	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9694
9695	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9696	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9697	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9698
9699	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9700	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9701	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9702	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9703	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9704	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9705	break;
9706	}
9707
9708	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9709	{
9710	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9711	AssertRCReturn(rc, rc);
9712	rc = EMInterpretCLTS(pVM, pVCpu);
9713	AssertRCReturn(rc, rc);
9714	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9715	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9716	Log4(("CRX CLTS write rc=%d\n", rc));
9717	break;
9718	}
9719
9720	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9721	{
9722	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9723	AssertRCReturn(rc, rc);
9724	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9725	if (RT_LIKELY(rc == VINF_SUCCESS))
9726	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9727	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9728	Log4(("CRX LMSW write rc=%d\n", rc));
9729	break;
9730	}
9731
9732	default:
9733	{
9734	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9735	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9736	}
9737	}
9738
9739	/* Validate possible error codes. */
9740	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9741	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9742	if (RT_SUCCESS(rc))
9743	{
9744	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9745	AssertRCReturn(rc2, rc2);
9746	}
9747
9748	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9749	return rc;
9750	}
9751
9752
9753	/**
9754	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9755	* VM-exit.
9756	*/
9757	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9758	{
9759	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9760	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9761
9762	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9763	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9764	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9765	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9766	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9767	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9768	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9769	AssertRCReturn(rc2, rc2);
9770
9771	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9772	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9773	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9774	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9775	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9776	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9777	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9778
9779	/* I/O operation lookup arrays. */
9780	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9781	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9782
9783	VBOXSTRICTRC rcStrict;
9784	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9785	const uint32_t cbInstr = pVmxTransient->cbInstr;
9786	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9787	PVM pVM = pVCpu->CTX_SUFF(pVM);
9788	if (fIOString)
9789	{
9790	/*
9791	* INS/OUTS - I/O String instruction.
9792	*
9793	* Use instruction-information if available, otherwise fall back on
9794	* interpreting the instruction.
9795	*/
9796	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9797	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9798	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9799	{
9800	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9801	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9802	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9803	AssertRCReturn(rc2, rc2);
9804	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9805	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9806	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9807	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9808	if (fIOWrite)
9809	{
9810	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9811	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9812	}
9813	else
9814	{
9815	/*
9816	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
9817	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
9818	* See Intel Instruction spec. for "INS".
9819	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
9820	*/
9821	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9822	}
9823	}
9824	else
9825	{
9826	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9827	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9828	AssertRCReturn(rc2, rc2);
9829	rcStrict = IEMExecOne(pVCpu);
9830	}
9831	/** @todo IEM needs to be setting these flags somehow. */
9832	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9833	fUpdateRipAlready = true;
9834	}
9835	else
9836	{
9837	/*
9838	* IN/OUT - I/O instruction.
9839	*/
9840	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9841	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9842	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9843	if (fIOWrite)
9844	{
9845	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9846	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9847	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9848	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9849	}
9850	else
9851	{
9852	uint32_t u32Result = 0;
9853	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9854	if (IOM_SUCCESS(rcStrict))
9855	{
9856	/* Save result of I/O IN instr. in AL/AX/EAX. */
9857	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9858	}
9859	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9860	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9861	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9862	}
9863	}
9864
9865	if (IOM_SUCCESS(rcStrict))
9866	{
9867	if (!fUpdateRipAlready)
9868	{
9869	pMixedCtx->rip += cbInstr;
9870	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9871	}
9872
9873	/*
9874	* If any I/O breakpoints are armed, we need to check if one triggered
9875	* and take appropriate action.
9876	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9877	*/
9878	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9879	AssertRCReturn(rc2, rc2);
9880
9881	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9882	* execution engines about whether hyper BPs and such are pending. */
9883	uint32_t const uDr7 = pMixedCtx->dr[7];
9884	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9885	&& X86_DR7_ANY_RW_IO(uDr7)
9886	&& (pMixedCtx->cr4 & X86_CR4_DE))
9887	\|\| DBGFBpIsHwIoArmed(pVM)))
9888	{
9889	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9890
9891	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
9892	VMMRZCallRing3Disable(pVCpu);
9893	HM_DISABLE_PREEMPT_IF_NEEDED();
9894
9895	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9896
9897	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9898	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9899	{
9900	/* Raise #DB. */
9901	if (fIsGuestDbgActive)
9902	ASMSetDR6(pMixedCtx->dr[6]);
9903	if (pMixedCtx->dr[7] != uDr7)
9904	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9905
9906	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9907	}
9908	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9909	else if ( rcStrict2 != VINF_SUCCESS
9910	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9911	rcStrict = rcStrict2;
9912
9913	HM_RESTORE_PREEMPT_IF_NEEDED();
9914	VMMRZCallRing3Enable(pVCpu);
9915	}
9916	}
9917
9918	#ifdef DEBUG
9919	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9920	Assert(!fIOWrite);
9921	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9922	Assert(fIOWrite);
9923	else
9924	{
9925	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9926	* statuses, that the VMM device and some others may return. See
9927	* IOM_SUCCESS() for guidance. */
9928	AssertMsg( RT_FAILURE(rcStrict)
9929	\|\| rcStrict == VINF_SUCCESS
9930	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9931	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9932	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9933	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9934	}
9935	#endif
9936
9937	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9938	return VBOXSTRICTRC_TODO(rcStrict);
9939	}
9940
9941
9942	/**
9943	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9944	* VM-exit.
9945	*/
9946	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9947	{
9948	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9949
9950	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9951	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9952	AssertRCReturn(rc, rc);
9953	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9954	{
9955	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9956	AssertRCReturn(rc, rc);
9957	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9958	{
9959	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9960
9961	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9962	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9963	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9964	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9965	{
9966	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9967	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9968
9969	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9970	Assert(!pVCpu->hm.s.Event.fPending);
9971	pVCpu->hm.s.Event.fPending = true;
9972	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
9973	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9974	AssertRCReturn(rc, rc);
9975	if (fErrorCodeValid)
9976	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9977	else
9978	pVCpu->hm.s.Event.u32ErrCode = 0;
9979	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9980	&& uVector == X86_XCPT_PF)
9981	{
9982	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9983	}
9984
9985	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9986	}
9987	}
9988	}
9989
9990	/** @todo Emulate task switch someday, currently just going back to ring-3 for
9991	* emulation. */
9992	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9993	return VERR_EM_INTERPRETER;
9994	}
9995
9996
9997	/**
9998	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9999	*/
10000	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10001	{
10002	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10003	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10004	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10005	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10006	AssertRCReturn(rc, rc);
10007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10008	return VINF_EM_DBG_STEPPED;
10009	}
10010
10011
10012	/**
10013	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10014	*/
10015	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10016	{
10017	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10018
10019	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10020	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10021	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10022	return VINF_SUCCESS;
10023	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10024	return rc;
10025
10026	#if 0
10027	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10028	* just sync the whole thing. */
10029	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10030	#else
10031	/* Aggressive state sync. for now. */
10032	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10033	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10034	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10035	#endif
10036	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10037	AssertRCReturn(rc, rc);
10038
10039	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10040	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10041	switch (uAccessType)
10042	{
10043	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10044	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10045	{
10046	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10047	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10048	{
10049	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10050	}
10051
10052	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10053	GCPhys &= PAGE_BASE_GC_MASK;
10054	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10055	PVM pVM = pVCpu->CTX_SUFF(pVM);
10056	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10057	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10058
10059	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10060	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10061	CPUMCTX2CORE(pMixedCtx), GCPhys);
10062	rc = VBOXSTRICTRC_VAL(rc2);
10063	Log4(("ApicAccess rc=%d\n", rc));
10064	if ( rc == VINF_SUCCESS
10065	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10066	\|\| rc == VERR_PAGE_NOT_PRESENT)
10067	{
10068	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10069	\| HM_CHANGED_GUEST_RSP
10070	\| HM_CHANGED_GUEST_RFLAGS
10071	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10072	rc = VINF_SUCCESS;
10073	}
10074	break;
10075	}
10076
10077	default:
10078	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10079	rc = VINF_EM_RAW_EMULATE_INSTR;
10080	break;
10081	}
10082
10083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10084	return rc;
10085	}
10086
10087
10088	/**
10089	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10090	* VM-exit.
10091	*/
10092	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10093	{
10094	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10095
10096	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10097	if (pVmxTransient->fWasGuestDebugStateActive)
10098	{
10099	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10100	HMVMX_RETURN_UNEXPECTED_EXIT();
10101	}
10102
10103	int rc = VERR_INTERNAL_ERROR_5;
10104	if ( !DBGFIsStepping(pVCpu)
10105	&& !pVCpu->hm.s.fSingleInstruction
10106	&& !pVmxTransient->fWasHyperDebugStateActive)
10107	{
10108	/* Don't intercept MOV DRx and #DB any more. */
10109	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10110	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10111	AssertRCReturn(rc, rc);
10112
10113	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10114	{
10115	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10116	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10117	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10118	AssertRCReturn(rc, rc);
10119	#endif
10120	}
10121
10122	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10123	VMMRZCallRing3Disable(pVCpu);
10124	HM_DISABLE_PREEMPT_IF_NEEDED();
10125
10126	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10127	PVM pVM = pVCpu->CTX_SUFF(pVM);
10128	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10129	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10130
10131	HM_RESTORE_PREEMPT_IF_NEEDED();
10132	VMMRZCallRing3Enable(pVCpu);
10133
10134	#ifdef VBOX_WITH_STATISTICS
10135	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10136	AssertRCReturn(rc, rc);
10137	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10138	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10139	else
10140	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10141	#endif
10142	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10143	return VINF_SUCCESS;
10144	}
10145
10146	/*
10147	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10148	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10149	*/
10150	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10151	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10152	AssertRCReturn(rc, rc);
10153	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10154
10155	PVM pVM = pVCpu->CTX_SUFF(pVM);
10156	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10157	{
10158	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10159	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10160	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10161	if (RT_SUCCESS(rc))
10162	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10163	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10164	}
10165	else
10166	{
10167	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10168	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10169	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10170	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10171	}
10172
10173	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10174	if (RT_SUCCESS(rc))
10175	{
10176	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10177	AssertRCReturn(rc2, rc2);
10178	}
10179	return rc;
10180	}
10181
10182
10183	/**
10184	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10185	* Conditional VM-exit.
10186	*/
10187	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10188	{
10189	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10190	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10191
10192	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10193	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10194	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10195	return VINF_SUCCESS;
10196	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10197	return rc;
10198
10199	RTGCPHYS GCPhys = 0;
10200	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10201
10202	#if 0
10203	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10204	#else
10205	/* Aggressive state sync. for now. */
10206	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10207	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10208	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10209	#endif
10210	AssertRCReturn(rc, rc);
10211
10212	/*
10213	* If we succeed, resume guest execution.
10214	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10215	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10216	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10217	* weird case. See @bugref{6043}.
10218	*/
10219	PVM pVM = pVCpu->CTX_SUFF(pVM);
10220	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10221	rc = VBOXSTRICTRC_VAL(rc2);
10222	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10223	if ( rc == VINF_SUCCESS
10224	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10225	\|\| rc == VERR_PAGE_NOT_PRESENT)
10226	{
10227	/* Successfully handled MMIO operation. */
10228	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10229	\| HM_CHANGED_GUEST_RSP
10230	\| HM_CHANGED_GUEST_RFLAGS
10231	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10232	rc = VINF_SUCCESS;
10233	}
10234	return rc;
10235	}
10236
10237
10238	/**
10239	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10240	* VM-exit.
10241	*/
10242	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10243	{
10244	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10245	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10246
10247	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10248	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10249	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10250	return VINF_SUCCESS;
10251	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10252	return rc;
10253
10254	RTGCPHYS GCPhys = 0;
10255	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10256	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10257	#if 0
10258	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10259	#else
10260	/* Aggressive state sync. for now. */
10261	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10262	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10263	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10264	#endif
10265	AssertRCReturn(rc, rc);
10266
10267	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10268	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10269
10270	RTGCUINT uErrorCode = 0;
10271	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10272	uErrorCode \|= X86_TRAP_PF_ID;
10273	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10274	uErrorCode \|= X86_TRAP_PF_RW;
10275	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10276	uErrorCode \|= X86_TRAP_PF_P;
10277
10278	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10279
10280	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10281	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10282
10283	/* Handle the pagefault trap for the nested shadow table. */
10284	PVM pVM = pVCpu->CTX_SUFF(pVM);
10285	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10286	TRPMResetTrap(pVCpu);
10287
10288	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10289	if ( rc == VINF_SUCCESS
10290	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10291	\|\| rc == VERR_PAGE_NOT_PRESENT)
10292	{
10293	/* Successfully synced our nested page tables. */
10294	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10295	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10296	\| HM_CHANGED_GUEST_RSP
10297	\| HM_CHANGED_GUEST_RFLAGS);
10298	return VINF_SUCCESS;
10299	}
10300
10301	Log4(("EPT return to ring-3 rc=%d\n"));
10302	return rc;
10303	}
10304
10305	/** @} */
10306
10307	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10308	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10309	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10310
10311	/** @name VM-exit exception handlers.
10312	* @{
10313	*/
10314
10315	/**
10316	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10317	*/
10318	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10319	{
10320	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10321	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10322
10323	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10324	AssertRCReturn(rc, rc);
10325
10326	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10327	{
10328	/* Old-style FPU error reporting needs some extra work. */
10329	/** @todo don't fall back to the recompiler, but do it manually. */
10330	return VERR_EM_INTERPRETER;
10331	}
10332
10333	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10334	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10335	return rc;
10336	}
10337
10338
10339	/**
10340	* VM-exit exception handler for #BP (Breakpoint exception).
10341	*/
10342	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10343	{
10344	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10345	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10346
10347	/** @todo Try optimize this by not saving the entire guest state unless
10348	* really needed. */
10349	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10350	AssertRCReturn(rc, rc);
10351
10352	PVM pVM = pVCpu->CTX_SUFF(pVM);
10353	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10354	if (rc == VINF_EM_RAW_GUEST_TRAP)
10355	{
10356	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10357	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10358	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10359	AssertRCReturn(rc, rc);
10360
10361	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10362	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10363	}
10364
10365	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10366	return rc;
10367	}
10368
10369
10370	/**
10371	* VM-exit exception handler for #DB (Debug exception).
10372	*/
10373	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10374	{
10375	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10376	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10377	Log6(("XcptDB\n"));
10378
10379	/*
10380	* Get the DR6-like values from the exit qualification and pass it to DBGF
10381	* for processing.
10382	*/
10383	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10384	AssertRCReturn(rc, rc);
10385
10386	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10387	uint64_t uDR6 = X86_DR6_INIT_VAL;
10388	uDR6 \|= ( pVmxTransient->uExitQualification
10389	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10390
10391	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10392	if (rc == VINF_EM_RAW_GUEST_TRAP)
10393	{
10394	/*
10395	* The exception was for the guest. Update DR6, DR7.GD and
10396	* IA32_DEBUGCTL.LBR before forwarding it.
10397	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10398	*/
10399	VMMRZCallRing3Disable(pVCpu);
10400	HM_DISABLE_PREEMPT_IF_NEEDED();
10401
10402	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10403	pMixedCtx->dr[6] \|= uDR6;
10404	if (CPUMIsGuestDebugStateActive(pVCpu))
10405	ASMSetDR6(pMixedCtx->dr[6]);
10406
10407	HM_RESTORE_PREEMPT_IF_NEEDED();
10408	VMMRZCallRing3Enable(pVCpu);
10409
10410	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10411	AssertRCReturn(rc, rc);
10412
10413	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10414	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10415
10416	/* Paranoia. */
10417	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10418	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10419
10420	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10421	AssertRCReturn(rc, rc);
10422
10423	/*
10424	* Raise #DB in the guest.
10425	*/
10426	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10427	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10428	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10429	AssertRCReturn(rc, rc);
10430	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10431	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10432	return VINF_SUCCESS;
10433	}
10434
10435	/*
10436	* Not a guest trap, must be a hypervisor related debug event then.
10437	* Update DR6 in case someone is interested in it.
10438	*/
10439	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10440	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10441	CPUMSetHyperDR6(pVCpu, uDR6);
10442
10443	return rc;
10444	}
10445
10446
10447	/**
10448	* VM-exit exception handler for #NM (Device-not-available exception: floating
10449	* point exception).
10450	*/
10451	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10452	{
10453	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10454
10455	/* We require CR0 and EFER. EFER is always up-to-date. */
10456	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10457	AssertRCReturn(rc, rc);
10458
10459	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10460	VMMRZCallRing3Disable(pVCpu);
10461	HM_DISABLE_PREEMPT_IF_NEEDED();
10462
10463	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10464	if (pVmxTransient->fWasGuestFPUStateActive)
10465	{
10466	rc = VINF_EM_RAW_GUEST_TRAP;
10467	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10468	}
10469	else
10470	{
10471	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10472	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10473	#endif
10474	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10475	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10476	}
10477
10478	HM_RESTORE_PREEMPT_IF_NEEDED();
10479	VMMRZCallRing3Enable(pVCpu);
10480
10481	if (rc == VINF_SUCCESS)
10482	{
10483	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10484	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10486	}
10487	else
10488	{
10489	/* Forward #NM to the guest. */
10490	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10491	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10492	AssertRCReturn(rc, rc);
10493	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10494	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
10495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10496	}
10497
10498	return VINF_SUCCESS;
10499	}
10500
10501
10502	/**
10503	* VM-exit exception handler for #GP (General-protection exception).
10504	*
10505	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10506	*/
10507	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10508	{
10509	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10510	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10511
10512	int rc = VERR_INTERNAL_ERROR_5;
10513	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10514	{
10515	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10516	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10517	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10518	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10519	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10520	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10521	AssertRCReturn(rc, rc);
10522	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10523	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10524	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10525	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10526	return rc;
10527	#else
10528	/* We don't intercept #GP. */
10529	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10530	return VERR_VMX_UNEXPECTED_EXCEPTION;
10531	#endif
10532	}
10533
10534	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10535	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10536
10537	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10538	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10539	AssertRCReturn(rc, rc);
10540
10541	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10542	uint32_t cbOp = 0;
10543	PVM pVM = pVCpu->CTX_SUFF(pVM);
10544	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10545	if (RT_SUCCESS(rc))
10546	{
10547	rc = VINF_SUCCESS;
10548	Assert(cbOp == pDis->cbInstr);
10549	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10550	switch (pDis->pCurInstr->uOpcode)
10551	{
10552	case OP_CLI:
10553	{
10554	pMixedCtx->eflags.Bits.u1IF = 0;
10555	pMixedCtx->rip += pDis->cbInstr;
10556	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10557	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10558	break;
10559	}
10560
10561	case OP_STI:
10562	{
10563	pMixedCtx->eflags.Bits.u1IF = 1;
10564	pMixedCtx->rip += pDis->cbInstr;
10565	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10566	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10567	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10569	break;
10570	}
10571
10572	case OP_HLT:
10573	{
10574	rc = VINF_EM_HALT;
10575	pMixedCtx->rip += pDis->cbInstr;
10576	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10578	break;
10579	}
10580
10581	case OP_POPF:
10582	{
10583	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10584	uint32_t cbParm = 0;
10585	uint32_t uMask = 0;
10586	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10587	{
10588	cbParm = 4;
10589	uMask = 0xffffffff;
10590	}
10591	else
10592	{
10593	cbParm = 2;
10594	uMask = 0xffff;
10595	}
10596
10597	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10598	RTGCPTR GCPtrStack = 0;
10599	X86EFLAGS Eflags;
10600	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10601	&GCPtrStack);
10602	if (RT_SUCCESS(rc))
10603	{
10604	Assert(sizeof(Eflags.u32) >= cbParm);
10605	Eflags.u32 = 0;
10606	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10607	}
10608	if (RT_FAILURE(rc))
10609	{
10610	rc = VERR_EM_INTERPRETER;
10611	break;
10612	}
10613	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10614	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10615	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10616	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10617	pMixedCtx->esp += cbParm;
10618	pMixedCtx->esp &= uMask;
10619	pMixedCtx->rip += pDis->cbInstr;
10620
10621	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10622	\| HM_CHANGED_GUEST_RSP
10623	\| HM_CHANGED_GUEST_RFLAGS);
10624	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10625	break;
10626	}
10627
10628	case OP_PUSHF:
10629	{
10630	uint32_t cbParm = 0;
10631	uint32_t uMask = 0;
10632	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10633	{
10634	cbParm = 4;
10635	uMask = 0xffffffff;
10636	}
10637	else
10638	{
10639	cbParm = 2;
10640	uMask = 0xffff;
10641	}
10642
10643	/* Get the stack pointer & push the contents of eflags onto the stack. */
10644	RTGCPTR GCPtrStack = 0;
10645	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10646	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10647	if (RT_FAILURE(rc))
10648	{
10649	rc = VERR_EM_INTERPRETER;
10650	break;
10651	}
10652	X86EFLAGS Eflags = pMixedCtx->eflags;
10653	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10654	Eflags.Bits.u1RF = 0;
10655	Eflags.Bits.u1VM = 0;
10656
10657	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10658	if (RT_FAILURE(rc))
10659	{
10660	rc = VERR_EM_INTERPRETER;
10661	break;
10662	}
10663	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10664	pMixedCtx->esp -= cbParm;
10665	pMixedCtx->esp &= uMask;
10666	pMixedCtx->rip += pDis->cbInstr;
10667	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
10668	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10669	break;
10670	}
10671
10672	case OP_IRET:
10673	{
10674	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10675	* instruction reference. */
10676	RTGCPTR GCPtrStack = 0;
10677	uint32_t uMask = 0xffff;
10678	uint16_t aIretFrame[3];
10679	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10680	{
10681	rc = VERR_EM_INTERPRETER;
10682	break;
10683	}
10684	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10685	&GCPtrStack);
10686	if (RT_SUCCESS(rc))
10687	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10688	if (RT_FAILURE(rc))
10689	{
10690	rc = VERR_EM_INTERPRETER;
10691	break;
10692	}
10693	pMixedCtx->eip = 0;
10694	pMixedCtx->ip = aIretFrame[0];
10695	pMixedCtx->cs.Sel = aIretFrame[1];
10696	pMixedCtx->cs.ValidSel = aIretFrame[1];
10697	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10698	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10699	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10700	pMixedCtx->sp += sizeof(aIretFrame);
10701	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10702	\| HM_CHANGED_GUEST_SEGMENT_REGS
10703	\| HM_CHANGED_GUEST_RSP
10704	\| HM_CHANGED_GUEST_RFLAGS);
10705	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10706	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10707	break;
10708	}
10709
10710	case OP_INT:
10711	{
10712	uint16_t uVector = pDis->Param1.uValue & 0xff;
10713	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10714	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10715	break;
10716	}
10717
10718	case OP_INTO:
10719	{
10720	if (pMixedCtx->eflags.Bits.u1OF)
10721	{
10722	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10724	}
10725	break;
10726	}
10727
10728	default:
10729	{
10730	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10731	EMCODETYPE_SUPERVISOR);
10732	rc = VBOXSTRICTRC_VAL(rc2);
10733	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10734	Log4(("#GP rc=%Rrc\n", rc));
10735	break;
10736	}
10737	}
10738	}
10739	else
10740	rc = VERR_EM_INTERPRETER;
10741
10742	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10743	("#GP Unexpected rc=%Rrc\n", rc));
10744	return rc;
10745	}
10746
10747
10748	/**
10749	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10750	* the exception reported in the VMX transient structure back into the VM.
10751	*
10752	* @remarks Requires uExitIntInfo in the VMX transient structure to be
10753	* up-to-date.
10754	*/
10755	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10756	{
10757	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10758
10759	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10760	hmR0VmxCheckExitDueToEventDelivery(). */
10761	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10762	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10763	AssertRCReturn(rc, rc);
10764	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10765
10766	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10767	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10768	return VINF_SUCCESS;
10769	}
10770
10771
10772	/**
10773	* VM-exit exception handler for #PF (Page-fault exception).
10774	*/
10775	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10776	{
10777	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10778	PVM pVM = pVCpu->CTX_SUFF(pVM);
10779	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10780	rc \|= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10781	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10782	AssertRCReturn(rc, rc);
10783
10784	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10785	if (pVM->hm.s.fNestedPaging)
10786	{
10787	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10788	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10789	{
10790	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10791	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10792	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
10793	}
10794	else
10795	{
10796	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10797	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10798	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10799	}
10800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10801	return rc;
10802	}
10803	#else
10804	Assert(!pVM->hm.s.fNestedPaging);
10805	#endif
10806
10807	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10808	AssertRCReturn(rc, rc);
10809
10810	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10811	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
10812
10813	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
10814	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
10815	(RTGCPTR)pVmxTransient->uExitQualification);
10816
10817	Log4(("#PF: rc=%Rrc\n", rc));
10818	if (rc == VINF_SUCCESS)
10819	{
10820	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10821	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10822	* memory? We don't update the whole state here... */
10823	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10824	\| HM_CHANGED_GUEST_RSP
10825	\| HM_CHANGED_GUEST_RFLAGS
10826	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10827	TRPMResetTrap(pVCpu);
10828	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10829	return rc;
10830	}
10831	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10832	{
10833	if (!pVmxTransient->fVectoringPF)
10834	{
10835	/* It's a guest page fault and needs to be reflected to the guest. */
10836	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10837	TRPMResetTrap(pVCpu);
10838	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10839	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10840	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10841	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10842	}
10843	else
10844	{
10845	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10846	TRPMResetTrap(pVCpu);
10847	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10848	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10849	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10850	}
10851
10852	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10853	return VINF_SUCCESS;
10854	}
10855
10856	TRPMResetTrap(pVCpu);
10857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10858	return rc;
10859	}
10860
10861	/** @} */
10862

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49074

Download in other formats: