HMVMXR0.cpp@ 49664

Last change on this file since 49664 was 49664, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: MSR optimizations for world-switch of 64-bit guests.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 470.4 KB

Line
1	/* $Id: HMVMXR0.cpp 49664 2013-11-26 15:53:35Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	# define HMVMX_SAVE_FULL_GUEST_STATE
40	# define HMVMX_SYNC_FULL_GUEST_STATE
41	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	# define HMVMX_ALWAYS_TRAP_PF
44	# define HMVMX_ALWAYS_SWAP_FPU_STATE
45	# define HMVMX_ALWAYS_FLUSH_TLB
46	#endif
47
48
49	/*******************************************************************************
50	* Defined Constants And Macros *
51	*******************************************************************************/
52	#if defined(RT_ARCH_AMD64)
53	# define HMVMX_IS_64BIT_HOST_MODE() (true)
54	typedef RTHCUINTREG HMVMXHCUINTREG;
55	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56	extern "C" uint32_t g_fVMXIs64bitHost;
57	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58	typedef uint64_t HMVMXHCUINTREG;
59	#else
60	# define HMVMX_IS_64BIT_HOST_MODE() (false)
61	typedef RTHCUINTREG HMVMXHCUINTREG;
62	#endif
63
64	/** Use the function table. */
65	#define HMVMX_USE_FUNCTION_TABLE
66
67	/** Determine which tagged-TLB flush handler to use. */
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
70	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
71	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
72
73	/** @name Updated-guest-state flags.
74	* @{ */
75	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
76	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
77	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
78	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
79	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
80	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
81	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
82	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
83	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
84	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
85	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
86	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
91	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
109	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
110	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113	/** @} */
114
115	/** @name
116	* Flags to skip redundant reads of some common VMCS fields that are not part of
117	* the guest-CPU state but are in the transient structure.
118	*/
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
126	/** @} */
127
128	/** @name
129	* States of the VMCS.
130	*
131	* This does not reflect all possible VMCS states but currently only those
132	* needed for maintaining the VMCS consistently even when thread-context hooks
133	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
134	*/
135	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
136	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
137	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
138	/** @} */
139
140	/**
141	* Exception bitmap mask for real-mode guests (real-on-v86).
142	*
143	* We need to intercept all exceptions manually (except #PF). #NM is also
144	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
145	* even in real-mode if we have Nested Paging support.
146	*/
147	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
148	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
149	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
150	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
151	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
153	\| RT_BIT(X86_XCPT_XF))
154
155	/**
156	* Exception bitmap mask for all contributory exceptions.
157	*
158	* Page fault is deliberately excluded here as it's conditional as to whether
159	* it's contributory or benign. Page faults are handled separately.
160	*/
161	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
162	\| RT_BIT(X86_XCPT_DE))
163
164	/** Maximum VM-instruction error number. */
165	#define HMVMX_INSTR_ERROR_MAX 28
166
167	/** Profiling macro. */
168	#ifdef HM_PROFILE_EXIT_DISPATCH
169	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171	#else
172	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174	#endif
175
176	/** Assert that preemption is disabled or covered by thread-context hooks. */
177	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
178	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180	/** Assert that we haven't migrated CPUs when thread-context hooks are not
181	* used. */
182	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184	("Illegal migration! Entered on CPU %u Current %u\n", \
185	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187	/** Helper macro for VM-exit handlers called unexpectedly. */
188	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189	do { \
190	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191	return VERR_VMX_UNEXPECTED_EXIT; \
192	} while (0)
193
194
195	/*******************************************************************************
196	* Structures and Typedefs *
197	*******************************************************************************/
198	/**
199	* VMX transient state.
200	*
201	* A state structure for holding miscellaneous information across
202	* VMX non-root operation and restored after the transition.
203	*/
204	typedef struct VMXTRANSIENT
205	{
206	/** The host's rflags/eflags. */
207	RTCCUINTREG uEflags;
208	#if HC_ARCH_BITS == 32
209	uint32_t u32Alignment0;
210	#endif
211	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
212	uint64_t u64LStarMsr;
213	/** The guest's TPR value used for TPR shadowing. */
214	uint8_t u8GuestTpr;
215	/** Alignment. */
216	uint8_t abAlignment0[7];
217
218	/** The basic VM-exit reason. */
219	uint16_t uExitReason;
220	/** Alignment. */
221	uint16_t u16Alignment0;
222	/** The VM-exit interruption error code. */
223	uint32_t uExitIntErrorCode;
224	/** The VM-exit exit qualification. */
225	uint64_t uExitQualification;
226
227	/** The VM-exit interruption-information field. */
228	uint32_t uExitIntInfo;
229	/** The VM-exit instruction-length field. */
230	uint32_t cbInstr;
231	/** The VM-exit instruction-information field. */
232	union
233	{
234	/** Plain unsigned int representation. */
235	uint32_t u;
236	/** INS and OUTS information. */
237	struct
238	{
239	uint32_t u6Reserved0 : 7;
240	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241	uint32_t u3AddrSize : 3;
242	uint32_t u5Reserved1 : 5;
243	/** The segment register (X86_SREG_XXX). */
244	uint32_t iSegReg : 3;
245	uint32_t uReserved2 : 14;
246	} StrIo;
247	} ExitInstrInfo;
248	/** Whether the VM-entry failed or not. */
249	bool fVMEntryFailed;
250	/** Alignment. */
251	uint8_t abAlignment1[3];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest FPU was active at the time of VM-exit. */
269	bool fWasGuestFPUStateActive;
270	/** Whether the guest debug state was active at the time of VM-exit. */
271	bool fWasGuestDebugStateActive;
272	/** Whether the hyper debug state was active at the time of VM-exit. */
273	bool fWasHyperDebugStateActive;
274	/** Whether TSC-offsetting should be setup before VM-entry. */
275	bool fUpdateTscOffsettingAndPreemptTimer;
276	/** Whether the VM-exit was caused by a page-fault during delivery of a
277	* contributory exception or a page-fault. */
278	bool fVectoringPF;
279	} VMXTRANSIENT;
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
284	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
285	/** Pointer to VMX transient state. */
286	typedef VMXTRANSIENT *PVMXTRANSIENT;
287
288
289	/**
290	* MSR-bitmap read permissions.
291	*/
292	typedef enum VMXMSREXITREAD
293	{
294	/** Reading this MSR causes a VM-exit. */
295	VMXMSREXIT_INTERCEPT_READ = 0xb,
296	/** Reading this MSR does not cause a VM-exit. */
297	VMXMSREXIT_PASSTHRU_READ
298	} VMXMSREXITREAD;
299	/** Pointer to MSR-bitmap read permissions. */
300	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
301
302	/**
303	* MSR-bitmap write permissions.
304	*/
305	typedef enum VMXMSREXITWRITE
306	{
307	/** Writing to this MSR causes a VM-exit. */
308	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
309	/** Writing to this MSR does not cause a VM-exit. */
310	VMXMSREXIT_PASSTHRU_WRITE
311	} VMXMSREXITWRITE;
312	/** Pointer to MSR-bitmap write permissions. */
313	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
314
315
316	/**
317	* VMX VM-exit handler.
318	*
319	* @returns VBox status code.
320	* @param pVCpu Pointer to the VMCPU.
321	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
322	* out-of-sync. Make sure to update the required
323	* fields before using them.
324	* @param pVmxTransient Pointer to the VMX-transient structure.
325	*/
326	#ifndef HMVMX_USE_FUNCTION_TABLE
327	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
328	#else
329	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330	/** Pointer to VM-exit handler. */
331	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
332	#endif
333
334
335	/*******************************************************************************
336	* Internal Functions *
337	*******************************************************************************/
338	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
339	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
340	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
341	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
342	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
343	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
344	#endif
345	#ifndef HMVMX_USE_FUNCTION_TABLE
346	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
347	# define HMVMX_EXIT_DECL static int
348	#else
349	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
350	#endif
351
352	/** @name VM-exit handlers.
353	* @{
354	*/
355	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
356	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
357	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
358	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
359	static FNVMXEXITHANDLER hmR0VmxExitSipi;
360	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitSmi;
362	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
364	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
365	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
366	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
367	static FNVMXEXITHANDLER hmR0VmxExitHlt;
368	static FNVMXEXITHANDLER hmR0VmxExitInvd;
369	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
370	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
371	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
372	static FNVMXEXITHANDLER hmR0VmxExitRsm;
373	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
374	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
375	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
376	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
377	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
379	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
380	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
381	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
382	static FNVMXEXITHANDLER hmR0VmxExitMwait;
383	static FNVMXEXITHANDLER hmR0VmxExitMtf;
384	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
385	static FNVMXEXITHANDLER hmR0VmxExitPause;
386	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
387	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
388	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
391	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
392	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
393	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
394	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
395	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
396	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
397	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
398	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
399	/** @} */
400
401	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
408	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409	#endif
410	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
411
412	/*******************************************************************************
413	* Global Variables *
414	*******************************************************************************/
415	#ifdef HMVMX_USE_FUNCTION_TABLE
416
417	/**
418	* VMX_EXIT dispatch table.
419	*/
420	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
421	{
422	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
423	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
424	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
425	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
426	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
427	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
428	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
429	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
430	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
431	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
432	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
433	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
434	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
435	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
436	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
437	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
438	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
439	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
440	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
441	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
442	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
443	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
444	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
445	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
446	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
447	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
448	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
449	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
450	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
451	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
452	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
453	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
454	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
455	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
456	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
457	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
458	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
459	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
460	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
461	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
462	/* 40 UNDEFINED */ hmR0VmxExitPause,
463	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
464	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
465	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
466	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
467	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
468	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
470	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
471	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
472	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
473	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
474	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
475	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
476	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
477	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
478	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
479	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
480	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
481	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
482	};
483	#endif /* HMVMX_USE_FUNCTION_TABLE */
484
485	#ifdef VBOX_STRICT
486	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
487	{
488	/* 0 */ "(Not Used)",
489	/* 1 */ "VMCALL executed in VMX root operation.",
490	/* 2 */ "VMCLEAR with invalid physical address.",
491	/* 3 */ "VMCLEAR with VMXON pointer.",
492	/* 4 */ "VMLAUNCH with non-clear VMCS.",
493	/* 5 */ "VMRESUME with non-launched VMCS.",
494	/* 6 */ "VMRESUME after VMXOFF",
495	/* 7 */ "VM entry with invalid control fields.",
496	/* 8 */ "VM entry with invalid host state fields.",
497	/* 9 */ "VMPTRLD with invalid physical address.",
498	/* 10 */ "VMPTRLD with VMXON pointer.",
499	/* 11 */ "VMPTRLD with incorrect revision identifier.",
500	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
501	/* 13 */ "VMWRITE to read-only VMCS component.",
502	/* 14 */ "(Not Used)",
503	/* 15 */ "VMXON executed in VMX root operation.",
504	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
505	/* 17 */ "VM entry with non-launched executing VMCS.",
506	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
507	/* 19 */ "VMCALL with non-clear VMCS.",
508	/* 20 */ "VMCALL with invalid VM-exit control fields.",
509	/* 21 */ "(Not Used)",
510	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
511	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
512	/* 24 */ "VMCALL with invalid SMM-monitor features.",
513	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
514	/* 26 */ "VM entry with events blocked by MOV SS.",
515	/* 27 */ "(Not Used)",
516	/* 28 */ "Invalid operand to INVEPT/INVVPID."
517	};
518	#endif /* VBOX_STRICT */
519
520
521
522	/**
523	* Updates the VM's last error record. If there was a VMX instruction error,
524	* reads the error data from the VMCS and updates VCPU's last error record as
525	* well.
526	*
527	* @param pVM Pointer to the VM.
528	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
529	* VERR_VMX_UNABLE_TO_START_VM or
530	* VERR_VMX_INVALID_VMCS_FIELD).
531	* @param rc The error code.
532	*/
533	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
534	{
535	AssertPtr(pVM);
536	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
537	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
538	{
539	AssertPtrReturnVoid(pVCpu);
540	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
541	}
542	pVM->hm.s.lLastError = rc;
543	}
544
545
546	/**
547	* Reads the VM-entry interruption-information field from the VMCS into the VMX
548	* transient structure.
549	*
550	* @returns VBox status code.
551	* @param pVmxTransient Pointer to the VMX transient structure.
552	*
553	* @remarks No-long-jump zone!!!
554	*/
555	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
556	{
557	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
558	AssertRCReturn(rc, rc);
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Reads the VM-entry exception error code field from the VMCS into
565	* the VMX transient structure.
566	*
567	* @returns VBox status code.
568	* @param pVmxTransient Pointer to the VMX transient structure.
569	*
570	* @remarks No-long-jump zone!!!
571	*/
572	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
573	{
574	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
575	AssertRCReturn(rc, rc);
576	return VINF_SUCCESS;
577	}
578
579
580	/**
581	* Reads the VM-entry exception error code field from the VMCS into
582	* the VMX transient structure.
583	*
584	* @returns VBox status code.
585	* @param pVmxTransient Pointer to the VMX transient structure.
586	*
587	* @remarks No-long-jump zone!!!
588	*/
589	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
590	{
591	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
592	AssertRCReturn(rc, rc);
593	return VINF_SUCCESS;
594	}
595
596
597	/**
598	* Reads the VM-exit interruption-information field from the VMCS into the VMX
599	* transient structure.
600	*
601	* @returns VBox status code.
602	* @param pVmxTransient Pointer to the VMX transient structure.
603	*/
604	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
605	{
606	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
607	{
608	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
609	AssertRCReturn(rc, rc);
610	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
611	}
612	return VINF_SUCCESS;
613	}
614
615
616	/**
617	* Reads the VM-exit interruption error code from the VMCS into the VMX
618	* transient structure.
619	*
620	* @returns VBox status code.
621	* @param pVmxTransient Pointer to the VMX transient structure.
622	*/
623	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
624	{
625	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
626	{
627	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
628	AssertRCReturn(rc, rc);
629	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
630	}
631	return VINF_SUCCESS;
632	}
633
634
635	/**
636	* Reads the VM-exit instruction length field from the VMCS into the VMX
637	* transient structure.
638	*
639	* @returns VBox status code.
640	* @param pVCpu Pointer to the VMCPU.
641	* @param pVmxTransient Pointer to the VMX transient structure.
642	*/
643	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
644	{
645	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
646	{
647	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
648	AssertRCReturn(rc, rc);
649	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
650	}
651	return VINF_SUCCESS;
652	}
653
654
655	/**
656	* Reads the VM-exit instruction-information field from the VMCS into
657	* the VMX transient structure.
658	*
659	* @returns VBox status code.
660	* @param pVmxTransient Pointer to the VMX transient structure.
661	*/
662	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
663	{
664	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
665	{
666	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
667	AssertRCReturn(rc, rc);
668	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
669	}
670	return VINF_SUCCESS;
671	}
672
673
674	/**
675	* Reads the exit qualification from the VMCS into the VMX transient structure.
676	*
677	* @returns VBox status code.
678	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
679	* case).
680	* @param pVmxTransient Pointer to the VMX transient structure.
681	*/
682	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
683	{
684	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
685	{
686	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
687	AssertRCReturn(rc, rc);
688	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
689	}
690	return VINF_SUCCESS;
691	}
692
693
694	/**
695	* Reads the IDT-vectoring information field from the VMCS into the VMX
696	* transient structure.
697	*
698	* @returns VBox status code.
699	* @param pVmxTransient Pointer to the VMX transient structure.
700	*
701	* @remarks No-long-jump zone!!!
702	*/
703	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
704	{
705	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
706	{
707	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
708	AssertRCReturn(rc, rc);
709	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
710	}
711	return VINF_SUCCESS;
712	}
713
714
715	/**
716	* Reads the IDT-vectoring error code from the VMCS into the VMX
717	* transient structure.
718	*
719	* @returns VBox status code.
720	* @param pVmxTransient Pointer to the VMX transient structure.
721	*/
722	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
723	{
724	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
725	{
726	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
727	AssertRCReturn(rc, rc);
728	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
729	}
730	return VINF_SUCCESS;
731	}
732
733
734	/**
735	* Enters VMX root mode operation on the current CPU.
736	*
737	* @returns VBox status code.
738	* @param pVM Pointer to the VM (optional, can be NULL, after
739	* a resume).
740	* @param HCPhysCpuPage Physical address of the VMXON region.
741	* @param pvCpuPage Pointer to the VMXON region.
742	*/
743	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
744	{
745	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
746	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
747	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
748
749	if (pVM)
750	{
751	/* Write the VMCS revision dword to the VMXON region. */
752	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
753	}
754
755	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
756	RTCCUINTREG uEflags = ASMIntDisableFlags();
757
758	/* Enable the VMX bit in CR4 if necessary. */
759	RTCCUINTREG uCr4 = ASMGetCR4();
760	if (!(uCr4 & X86_CR4_VMXE))
761	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
762
763	/* Enter VMX root mode. */
764	int rc = VMXEnable(HCPhysCpuPage);
765	if (RT_FAILURE(rc))
766	ASMSetCR4(uCr4);
767
768	/* Restore interrupts. */
769	ASMSetFlags(uEflags);
770	return rc;
771	}
772
773
774	/**
775	* Exits VMX root mode operation on the current CPU.
776	*
777	* @returns VBox status code.
778	*/
779	static int hmR0VmxLeaveRootMode(void)
780	{
781	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
782
783	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
784	RTCCUINTREG uEflags = ASMIntDisableFlags();
785
786	/* If we're for some reason not in VMX root mode, then don't leave it. */
787	RTCCUINTREG uHostCR4 = ASMGetCR4();
788
789	int rc;
790	if (uHostCR4 & X86_CR4_VMXE)
791	{
792	/* Exit VMX root mode and clear the VMX bit in CR4. */
793	VMXDisable();
794	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
795	rc = VINF_SUCCESS;
796	}
797	else
798	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
799
800	/* Restore interrupts. */
801	ASMSetFlags(uEflags);
802	return rc;
803	}
804
805
806	/**
807	* Allocates and maps one physically contiguous page. The allocated page is
808	* zero'd out. (Used by various VT-x structures).
809	*
810	* @returns IPRT status code.
811	* @param pMemObj Pointer to the ring-0 memory object.
812	* @param ppVirt Where to store the virtual address of the
813	* allocation.
814	* @param pPhys Where to store the physical address of the
815	* allocation.
816	*/
817	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
818	{
819	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
821	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
822
823	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
824	if (RT_FAILURE(rc))
825	return rc;
826	ppVirt = RTR0MemObjAddress(pMemObj);
827	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
828	ASMMemZero32(*ppVirt, PAGE_SIZE);
829	return VINF_SUCCESS;
830	}
831
832
833	/**
834	* Frees and unmaps an allocated physical page.
835	*
836	* @param pMemObj Pointer to the ring-0 memory object.
837	* @param ppVirt Where to re-initialize the virtual address of
838	* allocation as 0.
839	* @param pHCPhys Where to re-initialize the physical address of the
840	* allocation as 0.
841	*/
842	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
843	{
844	AssertPtr(pMemObj);
845	AssertPtr(ppVirt);
846	AssertPtr(pHCPhys);
847	if (*pMemObj != NIL_RTR0MEMOBJ)
848	{
849	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
850	AssertRC(rc);
851	*pMemObj = NIL_RTR0MEMOBJ;
852	*ppVirt = 0;
853	*pHCPhys = 0;
854	}
855	}
856
857
858	/**
859	* Worker function to free VT-x related structures.
860	*
861	* @returns IPRT status code.
862	* @param pVM Pointer to the VM.
863	*/
864	static void hmR0VmxStructsFree(PVM pVM)
865	{
866	for (VMCPUID i = 0; i < pVM->cCpus; i++)
867	{
868	PVMCPU pVCpu = &pVM->aCpus[i];
869	AssertPtr(pVCpu);
870
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
872	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873
874	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
875	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
876
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
878	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
879	}
880
881	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
882	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
883	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
884	#endif
885	}
886
887
888	/**
889	* Worker function to allocate VT-x related VM structures.
890	*
891	* @returns IPRT status code.
892	* @param pVM Pointer to the VM.
893	*/
894	static int hmR0VmxStructsAlloc(PVM pVM)
895	{
896	/*
897	* Initialize members up-front so we can cleanup properly on allocation failure.
898	*/
899	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
900	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
901	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
902	pVM->hm.s.vmx.HCPhys##a_Name = 0;
903
904	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
905	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
908
909	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
910	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
911	#endif
912	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
913
914	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
915	for (VMCPUID i = 0; i < pVM->cCpus; i++)
916	{
917	PVMCPU pVCpu = &pVM->aCpus[i];
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
922	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
923	}
924	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
925	#undef VMXLOCAL_INIT_VM_MEMOBJ
926
927	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
928	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
929	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
930	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
931
932	/*
933	* Allocate all the VT-x structures.
934	*/
935	int rc = VINF_SUCCESS;
936	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
937	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
938	if (RT_FAILURE(rc))
939	goto cleanup;
940	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
941	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
942	#endif
943
944	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
945	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
946	{
947	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
948	&pVM->hm.s.vmx.HCPhysApicAccess);
949	if (RT_FAILURE(rc))
950	goto cleanup;
951	}
952
953	/*
954	* Initialize per-VCPU VT-x structures.
955	*/
956	for (VMCPUID i = 0; i < pVM->cCpus; i++)
957	{
958	PVMCPU pVCpu = &pVM->aCpus[i];
959	AssertPtr(pVCpu);
960
961	/* Allocate the VM control structure (VMCS). */
962	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
963	if (RT_FAILURE(rc))
964	goto cleanup;
965
966	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
967	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
968	{
969	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
970	&pVCpu->hm.s.vmx.HCPhysVirtApic);
971	if (RT_FAILURE(rc))
972	goto cleanup;
973	}
974
975	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
976	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
977	{
978	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
979	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
980	if (RT_FAILURE(rc))
981	goto cleanup;
982	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
983	}
984
985	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
986	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
987	if (RT_FAILURE(rc))
988	goto cleanup;
989
990	/* Allocate the VM-exit MSR-load page for the host MSRs. */
991	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
992	if (RT_FAILURE(rc))
993	goto cleanup;
994	}
995
996	return VINF_SUCCESS;
997
998	cleanup:
999	hmR0VmxStructsFree(pVM);
1000	return rc;
1001	}
1002
1003
1004	/**
1005	* Does global VT-x initialization (called during module initialization).
1006	*
1007	* @returns VBox status code.
1008	*/
1009	VMMR0DECL(int) VMXR0GlobalInit(void)
1010	{
1011	#ifdef HMVMX_USE_FUNCTION_TABLE
1012	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1013	# ifdef VBOX_STRICT
1014	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1015	Assert(g_apfnVMExitHandlers[i]);
1016	# endif
1017	#endif
1018	return VINF_SUCCESS;
1019	}
1020
1021
1022	/**
1023	* Does global VT-x termination (called during module termination).
1024	*/
1025	VMMR0DECL(void) VMXR0GlobalTerm()
1026	{
1027	/* Nothing to do currently. */
1028	}
1029
1030
1031	/**
1032	* Sets up and activates VT-x on the current CPU.
1033	*
1034	* @returns VBox status code.
1035	* @param pCpu Pointer to the global CPU info struct.
1036	* @param pVM Pointer to the VM (can be NULL after a host resume
1037	* operation).
1038	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1039	* fEnabledByHost is true).
1040	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1041	* @a fEnabledByHost is true).
1042	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1043	* enable VT-x on the host.
1044	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1045	*/
1046	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1047	void *pvMsrs)
1048	{
1049	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1050	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1051	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1052
1053	/* Enable VT-x if it's not already enabled by the host. */
1054	if (!fEnabledByHost)
1055	{
1056	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1057	if (RT_FAILURE(rc))
1058	return rc;
1059	}
1060
1061	/*
1062	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1063	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1064	*/
1065	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1066	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1067	{
1068	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1069	pCpu->fFlushAsidBeforeUse = false;
1070	}
1071	else
1072	pCpu->fFlushAsidBeforeUse = true;
1073
1074	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1075	++pCpu->cTlbFlushes;
1076
1077	return VINF_SUCCESS;
1078	}
1079
1080
1081	/**
1082	* Deactivates VT-x on the current CPU.
1083	*
1084	* @returns VBox status code.
1085	* @param pCpu Pointer to the global CPU info struct.
1086	* @param pvCpuPage Pointer to the VMXON region.
1087	* @param HCPhysCpuPage Physical address of the VMXON region.
1088	*
1089	* @remarks This function should never be called when SUPR0EnableVTx() or
1090	* similar was used to enable VT-x on the host.
1091	*/
1092	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1093	{
1094	NOREF(pCpu);
1095	NOREF(pvCpuPage);
1096	NOREF(HCPhysCpuPage);
1097
1098	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1099	return hmR0VmxLeaveRootMode();
1100	}
1101
1102
1103	/**
1104	* Sets the permission bits for the specified MSR in the MSR bitmap.
1105	*
1106	* @param pVCpu Pointer to the VMCPU.
1107	* @param uMSR The MSR value.
1108	* @param enmRead Whether reading this MSR causes a VM-exit.
1109	* @param enmWrite Whether writing this MSR causes a VM-exit.
1110	*/
1111	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1112	{
1113	int32_t iBit;
1114	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1115
1116	/*
1117	* Layout:
1118	* 0x000 - 0x3ff - Low MSR read bits
1119	* 0x400 - 0x7ff - High MSR read bits
1120	* 0x800 - 0xbff - Low MSR write bits
1121	* 0xc00 - 0xfff - High MSR write bits
1122	*/
1123	if (uMsr <= 0x00001FFF)
1124	iBit = uMsr;
1125	else if ( uMsr >= 0xC0000000
1126	&& uMsr <= 0xC0001FFF)
1127	{
1128	iBit = (uMsr - 0xC0000000);
1129	pbMsrBitmap += 0x400;
1130	}
1131	else
1132	{
1133	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1134	return;
1135	}
1136
1137	Assert(iBit <= 0x1fff);
1138	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1139	ASMBitSet(pbMsrBitmap, iBit);
1140	else
1141	ASMBitClear(pbMsrBitmap, iBit);
1142
1143	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1144	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1145	else
1146	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1147	}
1148
1149
1150	#ifdef VBOX_STRICT
1151	/**
1152	* Gets the permission bits for the specified MSR in the MSR bitmap.
1153	*
1154	* @returns VBox status code.
1155	* @retval VINF_SUCCESS if the specified MSR is found.
1156	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1157	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1158	*
1159	* @param pVCpu Pointer to the VMCPU.
1160	* @param uMsr The MSR.
1161	* @param penmRead Where to store the read permissions.
1162	* @param penmWrite Where to store the write permissions.
1163	*/
1164	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1165	{
1166	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1167	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1168	int32_t iBit;
1169	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1170
1171	/* See hmR0VmxSetMsrPermission() for the layout. */
1172	if (uMsr <= 0x00001FFF)
1173	iBit = uMsr;
1174	else if ( uMsr >= 0xC0000000
1175	&& uMsr <= 0xC0001FFF)
1176	{
1177	iBit = (uMsr - 0xC0000000);
1178	pbMsrBitmap += 0x400;
1179	}
1180	else
1181	{
1182	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1183	return VERR_NOT_SUPPORTED;
1184	}
1185
1186	Assert(iBit <= 0x1fff);
1187	if (ASMBitTest(pbMsrBitmap, iBit))
1188	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1189	else
1190	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1191
1192	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1193	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1194	else
1195	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1196	return VINF_SUCCESS;
1197	}
1198	#endif /* VBOX_STRICT */
1199
1200
1201	/**
1202	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1203	* area.
1204	*
1205	* @returns VBox status code.
1206	* @param pVCpu Pointer to the VMCPU.
1207	* @param cMsrs The number of MSRs.
1208	*/
1209	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1210	{
1211	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1212	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1213	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1214	{
1215	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1216	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1217	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1218	}
1219
1220	/* Update number of guest MSRs to load/store across the world-switch. */
1221	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1223
1224	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1225	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1226
1227	/* Update the VCPU's copy of the MSR count. */
1228	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1229
1230	return VINF_SUCCESS;
1231	}
1232
1233
1234	/**
1235	* Adds a new (or updates the value of an existing) guest/host MSR
1236	* pair to be swapped during the world-switch as part of the
1237	* auto-load/store MSR area in the VMCS.
1238	*
1239	* @returns VBox status code.
1240	* @param pVCpu Pointer to the VMCPU.
1241	* @param uMsr The MSR.
1242	* @param uGuestMsr Value of the guest MSR.
1243	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1244	* necessary.
1245	*/
1246	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1247	{
1248	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1249	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1250	uint32_t i;
1251	for (i = 0; i < cMsrs; i++)
1252	{
1253	if (pGuestMsr->u32Msr == uMsr)
1254	break;
1255	pGuestMsr++;
1256	}
1257
1258	bool fAdded = false;
1259	if (i == cMsrs)
1260	{
1261	++cMsrs;
1262	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1263	AssertRCReturn(rc, rc);
1264
1265	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1266	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1267	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1268
1269	fAdded = true;
1270	}
1271
1272	/* Update the MSR values in the auto-load/store MSR area. */
1273	pGuestMsr->u32Msr = uMsr;
1274	pGuestMsr->u64Value = uGuestMsrValue;
1275
1276	/* Create/update the MSR slot in the host MSR area. */
1277	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1278	pHostMsr += i;
1279	pHostMsr->u32Msr = uMsr;
1280
1281	/*
1282	* Update the host MSR only when requested by the called AND when we're
1283	* adding it to the auto-load/store area. Otherwise, it would have been
1284	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1285	*/
1286	if ( fAdded
1287	&& fUpdateHostMsr)
1288	{
1289	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1290	}
1291
1292	return VINF_SUCCESS;
1293	}
1294
1295
1296	/**
1297	* Removes a guest/shost MSR pair to be swapped during the world-switch from the
1298	* auto-load/store MSR area in the VMCS.
1299	*
1300	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1301	* area.
1302	*
1303	* @returns VBox status code.
1304	* @param pVCpu Pointer to the VMCPU.
1305	* @param uMsr The MSR.
1306	*/
1307	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1308	{
1309	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1310	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1311	for (uint32_t i = 0; i < cMsrs; i++)
1312	{
1313	/* Find the MSR. */
1314	if (pGuestMsr->u32Msr == uMsr)
1315	{
1316	/* If it's the last MSR, simply reduce the count. */
1317	if (i == cMsrs - 1)
1318	{
1319	--cMsrs;
1320	break;
1321	}
1322
1323	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1324	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1325	pLastGuestMsr += cMsrs;
1326	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1327	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1328
1329	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1330	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	pLastHostMsr += cMsrs;
1332	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1333	pHostMsr->u64Value = pLastHostMsr->u64Value;
1334	--cMsrs;
1335	break;
1336	}
1337	pGuestMsr++;
1338	}
1339
1340	/* Update the VMCS if the count changed (meaning the MSR was found). */
1341	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1342	{
1343	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1344	AssertRCReturn(rc, rc);
1345
1346	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1347	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1348	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1349	}
1350
1351	return VINF_SUCCESS;
1352	}
1353
1354
1355	/**
1356	* Checks if the specified guest MSR is part of the auto-load/store area in
1357	* the VMCS.
1358	*
1359	* @returns true if found, false otherwise.
1360	* @param pVCpu Pointer to the VMCPU.
1361	* @param uMsr The MSR to find.
1362	*/
1363	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1364	{
1365	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1366	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1367
1368	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1369	{
1370	if (pGuestMsr->u32Msr == uMsr)
1371	return true;
1372	}
1373	return false;
1374	}
1375
1376
1377	/**
1378	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1379	*
1380	* @param pVCpu Pointer to the VMCPU.
1381	*
1382	* @remarks No-long-jump zone!!!
1383	*/
1384	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1385	{
1386	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1387	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1388	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1389	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1390
1391	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1392	{
1393	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1394	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1395	}
1396
1397	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1398	}
1399
1400
1401	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1402	/**
1403	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1404	* perform lazy restoration of the host MSRs while leaving VT-x.
1405	*
1406	* @param pVCpu Pointer to the VMCPU.
1407	*
1408	* @remarks No-long-jump zone!!!
1409	*/
1410	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1411	{
1412	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1413
1414	#define VMXLOCAL_SAVE_HOST_MSR(uMsr, a_HostMsrField, RestoreFlag) \
1415	do { \
1416	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag))) \
1417	{ \
1418	pVCpu->hm.s.vmx.u64Host##a_HostMsrField = ASMRdMsr(uMsr); \
1419	Log4(("hmR0VmxLazySaveHostMsrs: uMsr=%#RX32 HostValue=%#RX64\n", (uMsr), pVCpu->hm.s.vmx.u64Host##a_HostMsrField)); \
1420	} \
1421	} while (0)
1422
1423	/*
1424	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1425	*/
1426	VMXLOCAL_SAVE_HOST_MSR(MSR_K8_LSTAR, LStarMsr, VMX_RESTORE_HOST_MSR_LSTAR);
1427	VMXLOCAL_SAVE_HOST_MSR(MSR_K6_STAR, StarMsr, VMX_RESTORE_HOST_MSR_STAR);
1428	VMXLOCAL_SAVE_HOST_MSR(MSR_K8_SF_MASK, SFMaskMsr, VMX_RESTORE_HOST_MSR_SFMASK);
1429	VMXLOCAL_SAVE_HOST_MSR(MSR_K8_KERNEL_GS_BASE, KernelGSBaseMsr, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1430	#undef VMXLOCAL_SAVE_HOST_MSR
1431	}
1432
1433
1434	/**
1435	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1436	* lazily while leaving VT-x.
1437	*
1438	* @returns true if it does, false otherwise.
1439	* @param pVCpu Pointer to the VMCPU.
1440	* @param uMsr The MSR to check.
1441	*/
1442	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1443	{
1444	NOREF(pVCpu);
1445	switch (uMsr)
1446	{
1447	case MSR_K8_LSTAR:
1448	case MSR_K6_STAR:
1449	case MSR_K8_SF_MASK:
1450	case MSR_K8_KERNEL_GS_BASE:
1451	return true;
1452	}
1453	return false;
1454	}
1455
1456
1457	/**
1458	* Saves a set of guests MSRs back into the guest-CPU context.
1459	*
1460	* @param pVCpu Pointer to the VMCPU.
1461	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1462	* out-of-sync. Make sure to update the required fields
1463	* before using them.
1464	*
1465	* @remarks No-long-jump zone!!!
1466	*/
1467	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1468	{
1469	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1470	#define VMXLOCAL_SAVE_GUEST_MSR(uMsr, a_GuestMsrField, RestoreFlag) \
1471	do { \
1472	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag)) \
1473	{ \
1474	pMixedCtx->msr##a_GuestMsrField = ASMRdMsr(uMsr); \
1475	Log4(("hmR0VmxLazySaveGuestMsrs: uMsr=%#RX32 GuestValue=%#RX64\n", (uMsr), pMixedCtx->msr##a_GuestMsrField)); \
1476	} \
1477	} while (0)
1478
1479	VMXLOCAL_SAVE_GUEST_MSR(MSR_K8_LSTAR, LSTAR, VMX_RESTORE_HOST_MSR_LSTAR);
1480	VMXLOCAL_SAVE_GUEST_MSR(MSR_K6_STAR, STAR, VMX_RESTORE_HOST_MSR_STAR);
1481	VMXLOCAL_SAVE_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, VMX_RESTORE_HOST_MSR_SFMASK);
1482	VMXLOCAL_SAVE_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1483
1484	#undef VMXLOCAL_SAVE_GUEST_MSR
1485	}
1486
1487
1488	/**
1489	* Loads a set of guests MSRs to allow read/passthru to the guest.
1490	*
1491	* The name of this function is slightly confusing. This function does NOT
1492	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1493	* common prefix for functions dealing with "lazy restoration" of the shared
1494	* MSRs.
1495	*
1496	* @param pVCpu Pointer to the VMCPU.
1497	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1498	* out-of-sync. Make sure to update the required fields
1499	* before using them.
1500	*
1501	* @remarks No-long-jump zone!!!
1502	*/
1503	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1504	{
1505	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1506
1507	#define VMXLOCAL_LOAD_GUEST_MSR(uMsr, a_GuestMsrField, a_HostMsrField, RestoreFlag) \
1508	do { \
1509	if ( (pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag)) \
1510	\|\| pMixedCtx->msr##a_GuestMsrField != pVCpu->hm.s.vmx.u64Host##a_HostMsrField) \
1511	{ \
1512	ASMWrMsr((uMsr), pMixedCtx->msr##a_GuestMsrField); \
1513	} \
1514	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= (RestoreFlag); \
1515	Log4(("Load: MSRSWAP uMsr=%#RX32 GuestValue=%#RX64\n", (uMsr), pMixedCtx->msr##a_GuestMsrField)); \
1516	} while (0)
1517
1518	VMXLOCAL_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStarMsr, VMX_RESTORE_HOST_MSR_LSTAR);
1519	VMXLOCAL_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, StarMsr, VMX_RESTORE_HOST_MSR_STAR);
1520	VMXLOCAL_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMaskMsr, VMX_RESTORE_HOST_MSR_SFMASK);
1521	VMXLOCAL_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBaseMsr, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1522
1523	#undef VMXLOCAL_LOAD_GUEST_MSR
1524	}
1525
1526
1527	/**
1528	* Performs lazy restoration of the set of host MSRs if they were previously
1529	* loaded with guest MSR values.
1530	*
1531	* @param pVCpu Pointer to the VMCPU.
1532	*
1533	* @remarks No-long-jump zone!!!
1534	* @remarks The guest MSRs should have been saved back into the guest-CPU
1535	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1536	*/
1537	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1538	{
1539	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1540
1541	#define VMXLOCAL_RESTORE_HOST_MSR(uMsr, a_HostMsrField, RestoreFlag) \
1542	do { \
1543	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag)) \
1544	{ \
1545	ASMWrMsr((uMsr), pVCpu->hm.s.vmx.u64Host##a_HostMsrField); \
1546	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(RestoreFlag); \
1547	Log4(("hmR0VmxLazyRestoreHostMsrs: uMsr=%#RX32 HostValue=%#RX64\n", (uMsr), \
1548	pVCpu->hm.s.vmx.u64Host##a_HostMsrField)); \
1549	} \
1550	} while (0)
1551
1552	VMXLOCAL_RESTORE_HOST_MSR(MSR_K8_LSTAR, LStarMsr, VMX_RESTORE_HOST_MSR_LSTAR);
1553	VMXLOCAL_RESTORE_HOST_MSR(MSR_K6_STAR, StarMsr, VMX_RESTORE_HOST_MSR_STAR);
1554	VMXLOCAL_RESTORE_HOST_MSR(MSR_K8_SF_MASK, SFMaskMsr, VMX_RESTORE_HOST_MSR_SFMASK);
1555	VMXLOCAL_RESTORE_HOST_MSR(MSR_K8_KERNEL_GS_BASE, KernelGSBaseMsr, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1556
1557	#undef VMXLOCAL_RESTORE_HOST_MSR
1558	}
1559	#endif /* HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
1560
1561
1562	#ifdef VBOX_STRICT
1563	/**
1564	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1565	* VMCS are correct.
1566	*
1567	* @param pVCpu Pointer to the VMCPU.
1568	*/
1569	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1570	{
1571	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1572
1573	/* Verify MSR counts in the VMCS are what we think it should be. */
1574	uint32_t cMsrs;
1575	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1576	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1577
1578	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1579	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1580
1581	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1582	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1583
1584	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1585	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1586	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1587	{
1588	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1589	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1590	pGuestMsr->u32Msr));
1591
1592	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1593	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1594	pHostMsr->u64Value, u64Msr));
1595
1596	/* Verify that the permissions are as expected in the MSR bitmap. */
1597	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1598	{
1599	VMXMSREXITREAD enmRead;
1600	VMXMSREXITWRITE enmWrite;
1601	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1602	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1603	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1604	pGuestMsr->u32Msr));
1605	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1606	pGuestMsr->u32Msr));
1607	}
1608	}
1609	}
1610	# endif /* VBOX_STRICT */
1611
1612
1613	/**
1614	* Flushes the TLB using EPT.
1615	*
1616	* @returns VBox status code.
1617	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1618	* enmFlush).
1619	* @param enmFlush Type of flush.
1620	*
1621	* @remarks Caller is responsible for making sure this function is called only
1622	* when NestedPaging is supported and providing @a enmFlush that is
1623	* supported by the CPU.
1624	* @remarks Can be called with interrupts disabled.
1625	*/
1626	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1627	{
1628	uint64_t au64Descriptor[2];
1629	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1630	au64Descriptor[0] = 0;
1631	else
1632	{
1633	Assert(pVCpu);
1634	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1635	}
1636	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1637
1638	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1639	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1640	rc));
1641	if ( RT_SUCCESS(rc)
1642	&& pVCpu)
1643	{
1644	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1645	}
1646	}
1647
1648
1649	/**
1650	* Flushes the TLB using VPID.
1651	*
1652	* @returns VBox status code.
1653	* @param pVM Pointer to the VM.
1654	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1655	* enmFlush).
1656	* @param enmFlush Type of flush.
1657	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1658	* on @a enmFlush).
1659	*
1660	* @remarks Can be called with interrupts disabled.
1661	*/
1662	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1663	{
1664	NOREF(pVM);
1665	AssertPtr(pVM);
1666	Assert(pVM->hm.s.vmx.fVpid);
1667
1668	uint64_t au64Descriptor[2];
1669	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1670	{
1671	au64Descriptor[0] = 0;
1672	au64Descriptor[1] = 0;
1673	}
1674	else
1675	{
1676	AssertPtr(pVCpu);
1677	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1678	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1679	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1680	au64Descriptor[1] = GCPtr;
1681	}
1682
1683	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1684	AssertMsg(rc == VINF_SUCCESS,
1685	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1686	if ( RT_SUCCESS(rc)
1687	&& pVCpu)
1688	{
1689	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1690	}
1691	}
1692
1693
1694	/**
1695	* Invalidates a guest page by guest virtual address. Only relevant for
1696	* EPT/VPID, otherwise there is nothing really to invalidate.
1697	*
1698	* @returns VBox status code.
1699	* @param pVM Pointer to the VM.
1700	* @param pVCpu Pointer to the VMCPU.
1701	* @param GCVirt Guest virtual address of the page to invalidate.
1702	*/
1703	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1704	{
1705	AssertPtr(pVM);
1706	AssertPtr(pVCpu);
1707	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1708
1709	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1710	if (!fFlushPending)
1711	{
1712	/*
1713	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1714	* See @bugref{6043} and @bugref{6177}.
1715	*
1716	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1717	* function maybe called in a loop with individual addresses.
1718	*/
1719	if (pVM->hm.s.vmx.fVpid)
1720	{
1721	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1722	{
1723	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1724	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1725	}
1726	else
1727	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1728	}
1729	else if (pVM->hm.s.fNestedPaging)
1730	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1731	}
1732
1733	return VINF_SUCCESS;
1734	}
1735
1736
1737	/**
1738	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1739	* otherwise there is nothing really to invalidate.
1740	*
1741	* @returns VBox status code.
1742	* @param pVM Pointer to the VM.
1743	* @param pVCpu Pointer to the VMCPU.
1744	* @param GCPhys Guest physical address of the page to invalidate.
1745	*/
1746	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1747	{
1748	NOREF(pVM); NOREF(GCPhys);
1749	LogFlowFunc(("%RGp\n", GCPhys));
1750
1751	/*
1752	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1753	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1754	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1755	*/
1756	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1757	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1758	return VINF_SUCCESS;
1759	}
1760
1761
1762	/**
1763	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1764	* case where neither EPT nor VPID is supported by the CPU.
1765	*
1766	* @param pVM Pointer to the VM.
1767	* @param pVCpu Pointer to the VMCPU.
1768	* @param pCpu Pointer to the global HM struct.
1769	*
1770	* @remarks Called with interrupts disabled.
1771	*/
1772	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1773	{
1774	AssertPtr(pVCpu);
1775	AssertPtr(pCpu);
1776	NOREF(pVM);
1777
1778	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1779
1780	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1781	#if 0
1782	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1783	pVCpu->hm.s.TlbShootdown.cPages = 0;
1784	#endif
1785
1786	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1787	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1788	pVCpu->hm.s.fForceTLBFlush = false;
1789	return;
1790	}
1791
1792
1793	/**
1794	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1795	*
1796	* @param pVM Pointer to the VM.
1797	* @param pVCpu Pointer to the VMCPU.
1798	* @param pCpu Pointer to the global HM CPU struct.
1799	* @remarks All references to "ASID" in this function pertains to "VPID" in
1800	* Intel's nomenclature. The reason is, to avoid confusion in compare
1801	* statements since the host-CPU copies are named "ASID".
1802	*
1803	* @remarks Called with interrupts disabled.
1804	*/
1805	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1806	{
1807	#ifdef VBOX_WITH_STATISTICS
1808	bool fTlbFlushed = false;
1809	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1810	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1811	if (!fTlbFlushed) \
1812	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1813	} while (0)
1814	#else
1815	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1816	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1817	#endif
1818
1819	AssertPtr(pVM);
1820	AssertPtr(pCpu);
1821	AssertPtr(pVCpu);
1822	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1823	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1824	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1825
1826
1827	/*
1828	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1829	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1830	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1831	*/
1832	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1833	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1834	{
1835	++pCpu->uCurrentAsid;
1836	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1837	{
1838	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1839	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1840	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1841	}
1842
1843	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1844	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1845	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1846
1847	/*
1848	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1849	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1850	*/
1851	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1852	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1853	HMVMX_SET_TAGGED_TLB_FLUSHED();
1854	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1855	}
1856
1857	/* Check for explicit TLB shootdowns. */
1858	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1859	{
1860	/*
1861	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1862	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1863	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1864	* but not guest-physical mappings.
1865	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1866	*/
1867	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1868	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1869	HMVMX_SET_TAGGED_TLB_FLUSHED();
1870	}
1871
1872	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1873	* where it is commented out. Support individual entry flushing
1874	* someday. */
1875	#if 0
1876	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1877	{
1878	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1879
1880	/*
1881	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1882	* as supported by the CPU.
1883	*/
1884	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1885	{
1886	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1887	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1888	}
1889	else
1890	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1891
1892	HMVMX_SET_TAGGED_TLB_FLUSHED();
1893	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1894	pVCpu->hm.s.TlbShootdown.cPages = 0;
1895	}
1896	#endif
1897
1898	pVCpu->hm.s.fForceTLBFlush = false;
1899
1900	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1901
1902	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1903	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1904	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1905	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1906	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1907	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1908	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1909	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1910
1911	/* Update VMCS with the VPID. */
1912	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1913	AssertRC(rc);
1914
1915	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1916	}
1917
1918
1919	/**
1920	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1921	*
1922	* @returns VBox status code.
1923	* @param pVM Pointer to the VM.
1924	* @param pVCpu Pointer to the VMCPU.
1925	* @param pCpu Pointer to the global HM CPU struct.
1926	*
1927	* @remarks Called with interrupts disabled.
1928	*/
1929	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1930	{
1931	AssertPtr(pVM);
1932	AssertPtr(pVCpu);
1933	AssertPtr(pCpu);
1934	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1935	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1936
1937	/*
1938	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1939	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1940	*/
1941	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1942	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1943	{
1944	pVCpu->hm.s.fForceTLBFlush = true;
1945	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1946	}
1947
1948	/* Check for explicit TLB shootdown flushes. */
1949	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1950	{
1951	pVCpu->hm.s.fForceTLBFlush = true;
1952	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1953	}
1954
1955	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1956	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1957
1958	if (pVCpu->hm.s.fForceTLBFlush)
1959	{
1960	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1961	pVCpu->hm.s.fForceTLBFlush = false;
1962	}
1963	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1964	* where it is commented out. Support individual entry flushing
1965	* someday. */
1966	#if 0
1967	else
1968	{
1969	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1970	{
1971	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1972	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1973	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1974	}
1975	else
1976	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1977
1978	pVCpu->hm.s.TlbShootdown.cPages = 0;
1979	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1980	}
1981	#endif
1982	}
1983
1984
1985	/**
1986	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1987	*
1988	* @returns VBox status code.
1989	* @param pVM Pointer to the VM.
1990	* @param pVCpu Pointer to the VMCPU.
1991	* @param pCpu Pointer to the global HM CPU struct.
1992	*
1993	* @remarks Called with interrupts disabled.
1994	*/
1995	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1996	{
1997	AssertPtr(pVM);
1998	AssertPtr(pVCpu);
1999	AssertPtr(pCpu);
2000	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2001	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2002
2003	/*
2004	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2005	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2006	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2007	*/
2008	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2009	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2010	{
2011	pVCpu->hm.s.fForceTLBFlush = true;
2012	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2013	}
2014
2015	/* Check for explicit TLB shootdown flushes. */
2016	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2017	{
2018	/*
2019	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2020	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2021	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2022	*/
2023	pVCpu->hm.s.fForceTLBFlush = true;
2024	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2025	}
2026
2027	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2028	if (pVCpu->hm.s.fForceTLBFlush)
2029	{
2030	++pCpu->uCurrentAsid;
2031	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2032	{
2033	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2034	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2035	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2036	}
2037
2038	pVCpu->hm.s.fForceTLBFlush = false;
2039	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2040	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2041	if (pCpu->fFlushAsidBeforeUse)
2042	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2043	}
2044	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2045	* where it is commented out. Support individual entry flushing
2046	* someday. */
2047	#if 0
2048	else
2049	{
2050	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2051	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2052	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2053	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2054
2055	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2056	{
2057	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2058	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2059	{
2060	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2061	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2062	}
2063	else
2064	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2065
2066	pVCpu->hm.s.TlbShootdown.cPages = 0;
2067	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2068	}
2069	else
2070	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2071	}
2072	#endif
2073
2074	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2075	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2076	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2077	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2078	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2079	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2080
2081	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2082	AssertRC(rc);
2083	}
2084
2085
2086	/**
2087	* Flushes the guest TLB entry based on CPU capabilities.
2088	*
2089	* @param pVCpu Pointer to the VMCPU.
2090	* @param pCpu Pointer to the global HM CPU struct.
2091	*/
2092	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2093	{
2094	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2095	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2096	#endif
2097	PVM pVM = pVCpu->CTX_SUFF(pVM);
2098	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2099	{
2100	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2101	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2102	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2103	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2104	default:
2105	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2106	break;
2107	}
2108
2109	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2110	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2111
2112	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2113	}
2114
2115
2116	/**
2117	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2118	* TLB entries from the host TLB before VM-entry.
2119	*
2120	* @returns VBox status code.
2121	* @param pVM Pointer to the VM.
2122	*/
2123	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2124	{
2125	/*
2126	* Determine optimal flush type for Nested Paging.
2127	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2128	* guest execution (see hmR3InitFinalizeR0()).
2129	*/
2130	if (pVM->hm.s.fNestedPaging)
2131	{
2132	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2133	{
2134	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2135	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2136	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2137	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2138	else
2139	{
2140	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2143	}
2144
2145	/* Make sure the write-back cacheable memory type for EPT is supported. */
2146	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2147	{
2148	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2149	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2150	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2151	}
2152	}
2153	else
2154	{
2155	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2156	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2157	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2158	}
2159	}
2160
2161	/*
2162	* Determine optimal flush type for VPID.
2163	*/
2164	if (pVM->hm.s.vmx.fVpid)
2165	{
2166	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2167	{
2168	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2169	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2170	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2171	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2172	else
2173	{
2174	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2175	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2176	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2177	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2178	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2179	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2180	pVM->hm.s.vmx.fVpid = false;
2181	}
2182	}
2183	else
2184	{
2185	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2186	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2187	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2188	pVM->hm.s.vmx.fVpid = false;
2189	}
2190	}
2191
2192	/*
2193	* Setup the handler for flushing tagged-TLBs.
2194	*/
2195	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2196	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2197	else if (pVM->hm.s.fNestedPaging)
2198	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2199	else if (pVM->hm.s.vmx.fVpid)
2200	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2201	else
2202	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2203	return VINF_SUCCESS;
2204	}
2205
2206
2207	/**
2208	* Sets up pin-based VM-execution controls in the VMCS.
2209	*
2210	* @returns VBox status code.
2211	* @param pVM Pointer to the VM.
2212	* @param pVCpu Pointer to the VMCPU.
2213	*/
2214	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2215	{
2216	AssertPtr(pVM);
2217	AssertPtr(pVCpu);
2218
2219	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2220	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2221
2222	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2223	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2224	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2225
2226	/* Enable the VMX preemption timer. */
2227	if (pVM->hm.s.vmx.fUsePreemptTimer)
2228	{
2229	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2230	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2231	}
2232
2233	if ((val & zap) != val)
2234	{
2235	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2236	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2237	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2238	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2239	}
2240
2241	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2242	AssertRCReturn(rc, rc);
2243
2244	/* Update VCPU with the currently set pin-based VM-execution controls. */
2245	pVCpu->hm.s.vmx.u32PinCtls = val;
2246	return rc;
2247	}
2248
2249
2250	/**
2251	* Sets up processor-based VM-execution controls in the VMCS.
2252	*
2253	* @returns VBox status code.
2254	* @param pVM Pointer to the VM.
2255	* @param pVMCPU Pointer to the VMCPU.
2256	*/
2257	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2258	{
2259	AssertPtr(pVM);
2260	AssertPtr(pVCpu);
2261
2262	int rc = VERR_INTERNAL_ERROR_5;
2263	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2264	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2265
2266	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2267	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2268	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2269	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2270	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2271	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2272	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2273
2274	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2275	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2276	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2277	{
2278	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2279	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2280	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2281	}
2282
2283	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2284	if (!pVM->hm.s.fNestedPaging)
2285	{
2286	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2287	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2288	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2289	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2290	}
2291
2292	/* Use TPR shadowing if supported by the CPU. */
2293	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2294	{
2295	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2296	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2297	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2298	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2299	AssertRCReturn(rc, rc);
2300
2301	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2302	/* CR8 writes causes a VM-exit based on TPR threshold. */
2303	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2304	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2305	}
2306	else
2307	{
2308	/*
2309	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2310	* Set this control only for 64-bit guests.
2311	*/
2312	if (pVM->hm.s.fAllow64BitGuests)
2313	{
2314	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2315	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2316	}
2317	}
2318
2319	/* Use MSR-bitmaps if supported by the CPU. */
2320	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2321	{
2322	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2323
2324	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2325	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2326	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2327	AssertRCReturn(rc, rc);
2328
2329	/*
2330	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2331	* automatically as dedicated fields in the VMCS.
2332	*/
2333	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2334	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2335	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2336	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2337	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338
2339	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2340	/*
2341	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2342	*/
2343	if ( HMVMX_IS_64BIT_HOST_MODE()
2344	&& pVM->hm.s.fAllow64BitGuests)
2345	{
2346	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2347	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2348	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2349	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2350	}
2351	#endif
2352	}
2353
2354	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2355	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2356	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2357
2358	if ((val & zap) != val)
2359	{
2360	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2361	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2362	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2363	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2364	}
2365
2366	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2367	AssertRCReturn(rc, rc);
2368
2369	/* Update VCPU with the currently set processor-based VM-execution controls. */
2370	pVCpu->hm.s.vmx.u32ProcCtls = val;
2371
2372	/*
2373	* Secondary processor-based VM-execution controls.
2374	*/
2375	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2376	{
2377	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2378	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2379
2380	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2381	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2382
2383	if (pVM->hm.s.fNestedPaging)
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2385	else
2386	{
2387	/*
2388	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2389	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2390	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2391	*/
2392	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2393	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2394	}
2395
2396	if (pVM->hm.s.vmx.fVpid)
2397	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2398
2399	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2400	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2401
2402	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2403	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2404	* done dynamically. */
2405	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2406	{
2407	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2408	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2409	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2410	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2411	AssertRCReturn(rc, rc);
2412	}
2413
2414	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2415	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2416
2417	if ((val & zap) != val)
2418	{
2419	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2420	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2421	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2422	}
2423
2424	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2425	AssertRCReturn(rc, rc);
2426
2427	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2428	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2429	}
2430	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2431	{
2432	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2433	"available\n"));
2434	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2435	}
2436
2437	return VINF_SUCCESS;
2438	}
2439
2440
2441	/**
2442	* Sets up miscellaneous (everything other than Pin & Processor-based
2443	* VM-execution) control fields in the VMCS.
2444	*
2445	* @returns VBox status code.
2446	* @param pVM Pointer to the VM.
2447	* @param pVCpu Pointer to the VMCPU.
2448	*/
2449	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2450	{
2451	NOREF(pVM);
2452	AssertPtr(pVM);
2453	AssertPtr(pVCpu);
2454
2455	int rc = VERR_GENERAL_FAILURE;
2456
2457	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2458	#if 0
2459	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2460	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2461	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2462
2463	/*
2464	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2465	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2466	* We thus use the exception bitmap to control it rather than use both.
2467	*/
2468	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2469	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2470
2471	/** @todo Explore possibility of using IO-bitmaps. */
2472	/* All IO & IOIO instructions cause VM-exits. */
2473	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2474	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2475
2476	/* Initialize the MSR-bitmap area. */
2477	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2478	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2479	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2480	#endif
2481
2482	/* Setup MSR auto-load/store area. */
2483	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2484	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2485	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2486	AssertRCReturn(rc, rc);
2487	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2488	AssertRCReturn(rc, rc);
2489
2490	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2491	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2492	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2493	AssertRCReturn(rc, rc);
2494
2495	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2496	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2497	AssertRCReturn(rc, rc);
2498
2499	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2500	#if 0
2501	/* Setup debug controls */
2502	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2503	AssertRCReturn(rc, rc);
2504	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2505	AssertRCReturn(rc, rc);
2506	#endif
2507
2508	return rc;
2509	}
2510
2511
2512	/**
2513	* Sets up the initial exception bitmap in the VMCS based on static conditions
2514	* (i.e. conditions that cannot ever change after starting the VM).
2515	*
2516	* @returns VBox status code.
2517	* @param pVM Pointer to the VM.
2518	* @param pVCpu Pointer to the VMCPU.
2519	*/
2520	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2521	{
2522	AssertPtr(pVM);
2523	AssertPtr(pVCpu);
2524
2525	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2526
2527	uint32_t u32XcptBitmap = 0;
2528
2529	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2530	if (!pVM->hm.s.fNestedPaging)
2531	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2532
2533	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2534	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2535	AssertRCReturn(rc, rc);
2536	return rc;
2537	}
2538
2539
2540	/**
2541	* Sets up the initial guest-state mask. The guest-state mask is consulted
2542	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2543	* for the nested virtualization case (as it would cause a VM-exit).
2544	*
2545	* @param pVCpu Pointer to the VMCPU.
2546	*/
2547	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2548	{
2549	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2550	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2551	return VINF_SUCCESS;
2552	}
2553
2554
2555	/**
2556	* Does per-VM VT-x initialization.
2557	*
2558	* @returns VBox status code.
2559	* @param pVM Pointer to the VM.
2560	*/
2561	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2562	{
2563	LogFlowFunc(("pVM=%p\n", pVM));
2564
2565	int rc = hmR0VmxStructsAlloc(pVM);
2566	if (RT_FAILURE(rc))
2567	{
2568	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2569	return rc;
2570	}
2571
2572	return VINF_SUCCESS;
2573	}
2574
2575
2576	/**
2577	* Does per-VM VT-x termination.
2578	*
2579	* @returns VBox status code.
2580	* @param pVM Pointer to the VM.
2581	*/
2582	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2583	{
2584	LogFlowFunc(("pVM=%p\n", pVM));
2585
2586	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2587	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2588	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2589	#endif
2590	hmR0VmxStructsFree(pVM);
2591	return VINF_SUCCESS;
2592	}
2593
2594
2595	/**
2596	* Sets up the VM for execution under VT-x.
2597	* This function is only called once per-VM during initialization.
2598	*
2599	* @returns VBox status code.
2600	* @param pVM Pointer to the VM.
2601	*/
2602	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2603	{
2604	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2605	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2606
2607	LogFlowFunc(("pVM=%p\n", pVM));
2608
2609	/*
2610	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2611	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2612	*/
2613	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2614	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2615	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2616	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2617	{
2618	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2619	return VERR_INTERNAL_ERROR;
2620	}
2621
2622	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2623	/*
2624	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2625	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2626	*/
2627	if ( pVM->hm.s.fAllow64BitGuests
2628	&& !HMVMX_IS_64BIT_HOST_MODE())
2629	{
2630	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2631	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2632	}
2633	#endif
2634
2635	/* Initialize these always, see hmR3InitFinalizeR0().*/
2636	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2637	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2638
2639	/* Setup the tagged-TLB flush handlers. */
2640	int rc = hmR0VmxSetupTaggedTlb(pVM);
2641	if (RT_FAILURE(rc))
2642	{
2643	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2644	return rc;
2645	}
2646
2647	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2648	{
2649	PVMCPU pVCpu = &pVM->aCpus[i];
2650	AssertPtr(pVCpu);
2651	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2652
2653	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2654	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2655
2656	/* Set revision dword at the beginning of the VMCS structure. */
2657	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2658
2659	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2660	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2661	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2662	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2663
2664	/* Load this VMCS as the current VMCS. */
2665	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2666	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2667	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2668
2669	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2670	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2671	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2672
2673	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2674	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2675	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2676
2677	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2678	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2679	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2680
2681	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2682	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2683	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2684
2685	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2686	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2687	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2688
2689	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2690	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2691	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2692	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2693	#endif
2694
2695	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2696	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2697	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2698	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2699
2700	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2701
2702	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2703	}
2704
2705	return VINF_SUCCESS;
2706	}
2707
2708
2709	/**
2710	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2711	* the VMCS.
2712	*
2713	* @returns VBox status code.
2714	* @param pVM Pointer to the VM.
2715	* @param pVCpu Pointer to the VMCPU.
2716	*/
2717	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2718	{
2719	NOREF(pVM); NOREF(pVCpu);
2720
2721	RTCCUINTREG uReg = ASMGetCR0();
2722	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2723	AssertRCReturn(rc, rc);
2724
2725	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2726	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2727	if (HMVMX_IS_64BIT_HOST_MODE())
2728	{
2729	uint64_t uRegCR3 = HMR0Get64bitCR3();
2730	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2731	}
2732	else
2733	#endif
2734	{
2735	uReg = ASMGetCR3();
2736	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2737	}
2738	AssertRCReturn(rc, rc);
2739
2740	uReg = ASMGetCR4();
2741	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2742	AssertRCReturn(rc, rc);
2743	return rc;
2744	}
2745
2746
2747	#if HC_ARCH_BITS == 64
2748	/**
2749	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2750	* requirements. See hmR0VmxSaveHostSegmentRegs().
2751	*/
2752	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2753	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2754	{ \
2755	bool fValidSelector = true; \
2756	if ((selValue) & X86_SEL_LDT) \
2757	{ \
2758	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2759	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2760	} \
2761	if (fValidSelector) \
2762	{ \
2763	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2764	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2765	} \
2766	(selValue) = 0; \
2767	}
2768	#endif
2769
2770
2771	/**
2772	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2773	* the host-state area in the VMCS.
2774	*
2775	* @returns VBox status code.
2776	* @param pVM Pointer to the VM.
2777	* @param pVCpu Pointer to the VMCPU.
2778	*/
2779	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2780	{
2781	NOREF(pVM);
2782	int rc = VERR_INTERNAL_ERROR_5;
2783
2784	/*
2785	* Host DS, ES, FS and GS segment registers.
2786	*/
2787	#if HC_ARCH_BITS == 64
2788	RTSEL uSelDS = ASMGetDS();
2789	RTSEL uSelES = ASMGetES();
2790	RTSEL uSelFS = ASMGetFS();
2791	RTSEL uSelGS = ASMGetGS();
2792	#else
2793	RTSEL uSelDS = 0;
2794	RTSEL uSelES = 0;
2795	RTSEL uSelFS = 0;
2796	RTSEL uSelGS = 0;
2797	#endif
2798
2799	/* Recalculate which host-state bits need to be manually restored. */
2800	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2801
2802	/*
2803	* Host CS and SS segment registers.
2804	*/
2805	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2806	RTSEL uSelCS;
2807	RTSEL uSelSS;
2808	if (HMVMX_IS_64BIT_HOST_MODE())
2809	{
2810	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2811	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2812	}
2813	else
2814	{
2815	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2816	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2817	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2818	}
2819	#else
2820	RTSEL uSelCS = ASMGetCS();
2821	RTSEL uSelSS = ASMGetSS();
2822	#endif
2823
2824	/*
2825	* Host TR segment register.
2826	*/
2827	RTSEL uSelTR = ASMGetTR();
2828
2829	#if HC_ARCH_BITS == 64
2830	/*
2831	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2832	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2833	*/
2834	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2835	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2836	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2837	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2838	# undef VMXLOCAL_ADJUST_HOST_SEG
2839	#endif
2840
2841	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2842	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2843	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2844	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2845	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2846	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2847	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2848	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2849	Assert(uSelCS);
2850	Assert(uSelTR);
2851
2852	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2853	#if 0
2854	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2855	Assert(uSelSS != 0);
2856	#endif
2857
2858	/* Write these host selector fields into the host-state area in the VMCS. */
2859	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2860	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2861	#if HC_ARCH_BITS == 64
2862	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2863	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2864	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2865	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2866	#endif
2867	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2868
2869	/*
2870	* Host GDTR and IDTR.
2871	*/
2872	RTGDTR Gdtr;
2873	RT_ZERO(Gdtr);
2874	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2875	if (HMVMX_IS_64BIT_HOST_MODE())
2876	{
2877	X86XDTR64 Gdtr64;
2878	X86XDTR64 Idtr64;
2879	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2880	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2881	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2882
2883	Gdtr.cbGdt = Gdtr64.cb;
2884	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2885	}
2886	else
2887	#endif
2888	{
2889	RTIDTR Idtr;
2890	ASMGetGDTR(&Gdtr);
2891	ASMGetIDTR(&Idtr);
2892	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2893	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2894
2895	#if HC_ARCH_BITS == 64
2896	/*
2897	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2898	* maximum limit (0xffff) on every VM-exit.
2899	*/
2900	if (Gdtr.cbGdt != 0xffff)
2901	{
2902	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2903	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2904	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2905	}
2906
2907	/*
2908	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2909	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2910	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2911	*/
2912	if (Idtr.cbIdt < 0x0fff)
2913	{
2914	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2915	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2916	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2917	}
2918	#endif
2919	}
2920
2921	/*
2922	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2923	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2924	*/
2925	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2926	{
2927	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2928	return VERR_VMX_INVALID_HOST_STATE;
2929	}
2930
2931	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2932	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2933	if (HMVMX_IS_64BIT_HOST_MODE())
2934	{
2935	/* We need the 64-bit TR base for hybrid darwin. */
2936	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2937	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2938	}
2939	else
2940	#endif
2941	{
2942	uintptr_t uTRBase;
2943	#if HC_ARCH_BITS == 64
2944	uTRBase = X86DESC64_BASE(pDesc);
2945
2946	/*
2947	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2948	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2949	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2950	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2951	*
2952	* [1] See Intel spec. 3.5 "System Descriptor Types".
2953	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2954	*/
2955	Assert(pDesc->System.u4Type == 11);
2956	if ( pDesc->System.u16LimitLow != 0x67
2957	\|\| pDesc->System.u4LimitHigh)
2958	{
2959	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2960	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2961
2962	/* Store the GDTR here as we need it while restoring TR. */
2963	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2964	}
2965	#else
2966	uTRBase = X86DESC_BASE(pDesc);
2967	#endif
2968	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2969	}
2970	AssertRCReturn(rc, rc);
2971
2972	/*
2973	* Host FS base and GS base.
2974	*/
2975	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2976	if (HMVMX_IS_64BIT_HOST_MODE())
2977	{
2978	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2979	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2980	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2981	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2982
2983	# if HC_ARCH_BITS == 64
2984	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2985	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2986	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2987	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2988	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2989	# endif
2990	}
2991	#endif
2992	return rc;
2993	}
2994
2995
2996	/**
2997	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2998	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2999	* the host after every successful VM exit.
3000	*
3001	* @returns VBox status code.
3002	* @param pVM Pointer to the VM.
3003	* @param pVCpu Pointer to the VMCPU.
3004	*
3005	* @remarks No-long-jump zone!!!
3006	*/
3007	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3008	{
3009	NOREF(pVM);
3010
3011	AssertPtr(pVCpu);
3012	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3013
3014	int rc = VINF_SUCCESS;
3015	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3016	if ( HMVMX_IS_64BIT_HOST_MODE()
3017	&& pVM->hm.s.fAllow64BitGuests)
3018	{
3019	hmR0VmxLazySaveHostMsrs(pVCpu);
3020	}
3021	#endif
3022
3023	if (pVCpu->hm.s.vmx.cMsrs > 0)
3024	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3025
3026	/*
3027	* Host Sysenter MSRs.
3028	*/
3029	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3030	AssertRCReturn(rc, rc);
3031	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3032	if (HMVMX_IS_64BIT_HOST_MODE())
3033	{
3034	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3035	AssertRCReturn(rc, rc);
3036	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3037	}
3038	else
3039	{
3040	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3041	AssertRCReturn(rc, rc);
3042	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3043	}
3044	#elif HC_ARCH_BITS == 32
3045	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3046	AssertRCReturn(rc, rc);
3047	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3048	#else
3049	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3050	AssertRCReturn(rc, rc);
3051	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3052	#endif
3053	AssertRCReturn(rc, rc);
3054
3055	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3056	* hmR0VmxSetupExitCtls() !! */
3057	return rc;
3058	}
3059
3060
3061	/**
3062	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3063	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3064	* controls".
3065	*
3066	* @returns VBox status code.
3067	* @param pVCpu Pointer to the VMCPU.
3068	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3069	* out-of-sync. Make sure to update the required fields
3070	* before using them.
3071	*
3072	* @remarks No-long-jump zone!!!
3073	*/
3074	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3075	{
3076	int rc = VINF_SUCCESS;
3077	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3078	{
3079	PVM pVM = pVCpu->CTX_SUFF(pVM);
3080	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3081	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3082
3083	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3084	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3085
3086	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3087	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3088	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3089	else
3090	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3091
3092	/*
3093	* The following should -not- be set (since we're not in SMM mode):
3094	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3095	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3096	*/
3097
3098	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3099	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3100	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3101
3102	if ((val & zap) != val)
3103	{
3104	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3105	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3106	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3107	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3108	}
3109
3110	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3111	AssertRCReturn(rc, rc);
3112
3113	/* Update VCPU with the currently set VM-exit controls. */
3114	pVCpu->hm.s.vmx.u32EntryCtls = val;
3115	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3116	}
3117	return rc;
3118	}
3119
3120
3121	/**
3122	* Sets up the VM-exit controls in the VMCS.
3123	*
3124	* @returns VBox status code.
3125	* @param pVM Pointer to the VM.
3126	* @param pVCpu Pointer to the VMCPU.
3127	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3128	* out-of-sync. Make sure to update the required fields
3129	* before using them.
3130	*
3131	* @remarks requires EFER.
3132	*/
3133	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3134	{
3135	NOREF(pMixedCtx);
3136
3137	int rc = VINF_SUCCESS;
3138	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3139	{
3140	PVM pVM = pVCpu->CTX_SUFF(pVM);
3141	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3142	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3143
3144	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3145	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3146
3147	/*
3148	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3149	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3150	*/
3151	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3152	if (HMVMX_IS_64BIT_HOST_MODE())
3153	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3154	else
3155	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3156	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3157	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3158	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3159	else
3160	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3161	#endif
3162
3163	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3164	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3165
3166	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3167	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3168	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3169	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3170	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3171
3172	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3173	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3174
3175	if ((val & zap) != val)
3176	{
3177	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3178	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3179	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3180	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3181	}
3182
3183	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3184	AssertRCReturn(rc, rc);
3185
3186	/* Update VCPU with the currently set VM-exit controls. */
3187	pVCpu->hm.s.vmx.u32ExitCtls = val;
3188	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3189	}
3190	return rc;
3191	}
3192
3193
3194	/**
3195	* Loads the guest APIC and related state.
3196	*
3197	* @returns VBox status code.
3198	* @param pVM Pointer to the VM.
3199	* @param pVCpu Pointer to the VMCPU.
3200	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3201	* out-of-sync. Make sure to update the required fields
3202	* before using them.
3203	*/
3204	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3205	{
3206	NOREF(pMixedCtx);
3207
3208	int rc = VINF_SUCCESS;
3209	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3210	{
3211	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3212	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3213	{
3214	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3215
3216	bool fPendingIntr = false;
3217	uint8_t u8Tpr = 0;
3218	uint8_t u8PendingIntr = 0;
3219	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3220	AssertRCReturn(rc, rc);
3221
3222	/*
3223	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3224	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3225	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3226	* the interrupt when we VM-exit for other reasons.
3227	*/
3228	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3229	uint32_t u32TprThreshold = 0;
3230	if (fPendingIntr)
3231	{
3232	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3233	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3234	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3235	if (u8PendingPriority <= u8TprPriority)
3236	u32TprThreshold = u8PendingPriority;
3237	else
3238	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3239	}
3240	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3241
3242	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3243	AssertRCReturn(rc, rc);
3244	}
3245
3246	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3247	}
3248	return rc;
3249	}
3250
3251
3252	/**
3253	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3254	*
3255	* @returns Guest's interruptibility-state.
3256	* @param pVCpu Pointer to the VMCPU.
3257	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3258	* out-of-sync. Make sure to update the required fields
3259	* before using them.
3260	*
3261	* @remarks No-long-jump zone!!!
3262	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3263	*/
3264	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3265	{
3266	/*
3267	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3268	* inhibit interrupts or clear any existing interrupt-inhibition.
3269	*/
3270	uint32_t uIntrState = 0;
3271	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3272	{
3273	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3274	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
3275	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3276	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3277	{
3278	/*
3279	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3280	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3281	*/
3282	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3283	}
3284	else if (pMixedCtx->eflags.Bits.u1IF)
3285	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3286	else
3287	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3288	}
3289	return uIntrState;
3290	}
3291
3292
3293	/**
3294	* Loads the guest's interruptibility-state into the guest-state area in the
3295	* VMCS.
3296	*
3297	* @returns VBox status code.
3298	* @param pVCpu Pointer to the VMCPU.
3299	* @param uIntrState The interruptibility-state to set.
3300	*/
3301	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3302	{
3303	NOREF(pVCpu);
3304	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3305	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3306	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3307	AssertRCReturn(rc, rc);
3308	return rc;
3309	}
3310
3311
3312	/**
3313	* Loads the guest's RIP into the guest-state area in the VMCS.
3314	*
3315	* @returns VBox status code.
3316	* @param pVCpu Pointer to the VMCPU.
3317	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3318	* out-of-sync. Make sure to update the required fields
3319	* before using them.
3320	*
3321	* @remarks No-long-jump zone!!!
3322	*/
3323	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3324	{
3325	int rc = VINF_SUCCESS;
3326	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3327	{
3328	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3329	AssertRCReturn(rc, rc);
3330
3331	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3332	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3333	}
3334	return rc;
3335	}
3336
3337
3338	/**
3339	* Loads the guest's RSP into the guest-state area in the VMCS.
3340	*
3341	* @returns VBox status code.
3342	* @param pVCpu Pointer to the VMCPU.
3343	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3344	* out-of-sync. Make sure to update the required fields
3345	* before using them.
3346	*
3347	* @remarks No-long-jump zone!!!
3348	*/
3349	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3350	{
3351	int rc = VINF_SUCCESS;
3352	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3353	{
3354	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3355	AssertRCReturn(rc, rc);
3356
3357	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3358	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3359	}
3360	return rc;
3361	}
3362
3363
3364	/**
3365	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3366	*
3367	* @returns VBox status code.
3368	* @param pVCpu Pointer to the VMCPU.
3369	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3370	* out-of-sync. Make sure to update the required fields
3371	* before using them.
3372	*
3373	* @remarks No-long-jump zone!!!
3374	*/
3375	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3376	{
3377	int rc = VINF_SUCCESS;
3378	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3379	{
3380	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3381	Let us assert it as such and use 32-bit VMWRITE. */
3382	Assert(!(pMixedCtx->rflags.u64 >> 32));
3383	X86EFLAGS Eflags = pMixedCtx->eflags;
3384	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3385	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3386
3387	/*
3388	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3389	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3390	*/
3391	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3392	{
3393	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3394	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3395	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3396	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3397	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3398	}
3399
3400	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3401	AssertRCReturn(rc, rc);
3402
3403	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3404	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3405	}
3406	return rc;
3407	}
3408
3409
3410	/**
3411	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3412	*
3413	* @returns VBox status code.
3414	* @param pVCpu Pointer to the VMCPU.
3415	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3416	* out-of-sync. Make sure to update the required fields
3417	* before using them.
3418	*
3419	* @remarks No-long-jump zone!!!
3420	*/
3421	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3422	{
3423	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3424	AssertRCReturn(rc, rc);
3425	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3426	AssertRCReturn(rc, rc);
3427	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3428	AssertRCReturn(rc, rc);
3429	return rc;
3430	}
3431
3432
3433	/**
3434	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3435	* CR0 is partially shared with the host and we have to consider the FPU bits.
3436	*
3437	* @returns VBox status code.
3438	* @param pVM Pointer to the VM.
3439	* @param pVCpu Pointer to the VMCPU.
3440	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3441	* out-of-sync. Make sure to update the required fields
3442	* before using them.
3443	*
3444	* @remarks No-long-jump zone!!!
3445	*/
3446	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3447	{
3448	/*
3449	* Guest CR0.
3450	* Guest FPU.
3451	*/
3452	int rc = VINF_SUCCESS;
3453	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3454	{
3455	Assert(!(pMixedCtx->cr0 >> 32));
3456	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3457	PVM pVM = pVCpu->CTX_SUFF(pVM);
3458
3459	/* The guest's view (read access) of its CR0 is unblemished. */
3460	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3461	AssertRCReturn(rc, rc);
3462	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3463
3464	/* Setup VT-x's view of the guest CR0. */
3465	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3466	if (pVM->hm.s.fNestedPaging)
3467	{
3468	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3469	{
3470	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3471	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3472	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3473	}
3474	else
3475	{
3476	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3477	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3478	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3479	}
3480
3481	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3482	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3483	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3484
3485	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3486	AssertRCReturn(rc, rc);
3487	}
3488	else
3489	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3490
3491	/*
3492	* Guest FPU bits.
3493	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3494	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3495	*/
3496	u32GuestCR0 \|= X86_CR0_NE;
3497	bool fInterceptNM = false;
3498	if (CPUMIsGuestFPUStateActive(pVCpu))
3499	{
3500	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3501	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3502	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3503	}
3504	else
3505	{
3506	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3507	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3508	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3509	}
3510
3511	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3512	bool fInterceptMF = false;
3513	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3514	fInterceptMF = true;
3515
3516	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3517	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3518	{
3519	Assert(PDMVmmDevHeapIsEnabled(pVM));
3520	Assert(pVM->hm.s.vmx.pRealModeTSS);
3521	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3522	fInterceptNM = true;
3523	fInterceptMF = true;
3524	}
3525	else
3526	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3527
3528	if (fInterceptNM)
3529	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3530	else
3531	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3532
3533	if (fInterceptMF)
3534	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3535	else
3536	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3537
3538	/* Additional intercepts for debugging, define these yourself explicitly. */
3539	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3540	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3541	\| RT_BIT(X86_XCPT_BP)
3542	\| RT_BIT(X86_XCPT_DB)
3543	\| RT_BIT(X86_XCPT_DE)
3544	\| RT_BIT(X86_XCPT_NM)
3545	\| RT_BIT(X86_XCPT_UD)
3546	\| RT_BIT(X86_XCPT_NP)
3547	\| RT_BIT(X86_XCPT_SS)
3548	\| RT_BIT(X86_XCPT_GP)
3549	\| RT_BIT(X86_XCPT_PF)
3550	\| RT_BIT(X86_XCPT_MF)
3551	;
3552	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3553	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3554	#endif
3555
3556	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3557
3558	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3559	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3560	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3561	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3562	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3563	else
3564	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3565
3566	u32GuestCR0 \|= uSetCR0;
3567	u32GuestCR0 &= uZapCR0;
3568	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3569
3570	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3571	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3572	AssertRCReturn(rc, rc);
3573	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3574	AssertRCReturn(rc, rc);
3575	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3576
3577	/*
3578	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3579	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3580	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3581	*/
3582	uint32_t u32CR0Mask = 0;
3583	u32CR0Mask = X86_CR0_PE
3584	\| X86_CR0_NE
3585	\| X86_CR0_WP
3586	\| X86_CR0_PG
3587	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3588	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3589	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3590
3591	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3592	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3593	* and @bugref{6944}. */
3594	#if 0
3595	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3596	u32CR0Mask &= ~X86_CR0_PE;
3597	#endif
3598	if (pVM->hm.s.fNestedPaging)
3599	u32CR0Mask &= ~X86_CR0_WP;
3600
3601	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3602	if (fInterceptNM)
3603	{
3604	u32CR0Mask \|= X86_CR0_TS
3605	\| X86_CR0_MP;
3606	}
3607
3608	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3609	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3610	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3611	AssertRCReturn(rc, rc);
3612	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3613
3614	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3615	}
3616	return rc;
3617	}
3618
3619
3620	/**
3621	* Loads the guest control registers (CR3, CR4) into the guest-state area
3622	* in the VMCS.
3623	*
3624	* @returns VBox status code.
3625	* @param pVM Pointer to the VM.
3626	* @param pVCpu Pointer to the VMCPU.
3627	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3628	* out-of-sync. Make sure to update the required fields
3629	* before using them.
3630	*
3631	* @remarks No-long-jump zone!!!
3632	*/
3633	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3634	{
3635	int rc = VINF_SUCCESS;
3636	PVM pVM = pVCpu->CTX_SUFF(pVM);
3637
3638	/*
3639	* Guest CR2.
3640	* It's always loaded in the assembler code. Nothing to do here.
3641	*/
3642
3643	/*
3644	* Guest CR3.
3645	*/
3646	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3647	{
3648	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3649	if (pVM->hm.s.fNestedPaging)
3650	{
3651	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3652
3653	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3654	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3655	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3656	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3657
3658	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3659	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3660	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3661
3662	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3663	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3664	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3665	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3666
3667	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3668	AssertRCReturn(rc, rc);
3669	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3670
3671	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3672	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3673	{
3674	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3675	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3676	{
3677	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3678	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3679	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3680	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3681	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3682	}
3683
3684	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3685	have Unrestricted Execution to handle the guest when it's not using paging. */
3686	GCPhysGuestCR3 = pMixedCtx->cr3;
3687	}
3688	else
3689	{
3690	/*
3691	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3692	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3693	* EPT takes care of translating it to host-physical addresses.
3694	*/
3695	RTGCPHYS GCPhys;
3696	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3697	Assert(PDMVmmDevHeapIsEnabled(pVM));
3698
3699	/* We obtain it here every time as the guest could have relocated this PCI region. */
3700	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3701	AssertRCReturn(rc, rc);
3702
3703	GCPhysGuestCR3 = GCPhys;
3704	}
3705
3706	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3707	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3708	}
3709	else
3710	{
3711	/* Non-nested paging case, just use the hypervisor's CR3. */
3712	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3713
3714	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3715	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3716	}
3717	AssertRCReturn(rc, rc);
3718
3719	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3720	}
3721
3722	/*
3723	* Guest CR4.
3724	*/
3725	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3726	{
3727	Assert(!(pMixedCtx->cr4 >> 32));
3728	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3729
3730	/* The guest's view of its CR4 is unblemished. */
3731	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3732	AssertRCReturn(rc, rc);
3733	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3734
3735	/* Setup VT-x's view of the guest CR4. */
3736	/*
3737	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3738	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3739	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3740	*/
3741	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3742	{
3743	Assert(pVM->hm.s.vmx.pRealModeTSS);
3744	Assert(PDMVmmDevHeapIsEnabled(pVM));
3745	u32GuestCR4 &= ~X86_CR4_VME;
3746	}
3747
3748	if (pVM->hm.s.fNestedPaging)
3749	{
3750	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3751	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3752	{
3753	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3754	u32GuestCR4 \|= X86_CR4_PSE;
3755	/* Our identity mapping is a 32-bit page directory. */
3756	u32GuestCR4 &= ~X86_CR4_PAE;
3757	}
3758	/* else use guest CR4.*/
3759	}
3760	else
3761	{
3762	/*
3763	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3764	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3765	*/
3766	switch (pVCpu->hm.s.enmShadowMode)
3767	{
3768	case PGMMODE_REAL: /* Real-mode. */
3769	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3770	case PGMMODE_32_BIT: /* 32-bit paging. */
3771	{
3772	u32GuestCR4 &= ~X86_CR4_PAE;
3773	break;
3774	}
3775
3776	case PGMMODE_PAE: /* PAE paging. */
3777	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3778	{
3779	u32GuestCR4 \|= X86_CR4_PAE;
3780	break;
3781	}
3782
3783	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3784	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3785	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3786	break;
3787	#endif
3788	default:
3789	AssertFailed();
3790	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3791	}
3792	}
3793
3794	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3795	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3796	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3797	u32GuestCR4 \|= uSetCR4;
3798	u32GuestCR4 &= uZapCR4;
3799
3800	/* Write VT-x's view of the guest CR4 into the VMCS. */
3801	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3802	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3803	AssertRCReturn(rc, rc);
3804
3805	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3806	uint32_t u32CR4Mask = 0;
3807	u32CR4Mask = X86_CR4_VME
3808	\| X86_CR4_PAE
3809	\| X86_CR4_PGE
3810	\| X86_CR4_PSE
3811	\| X86_CR4_VMXE;
3812	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3813	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3814	AssertRCReturn(rc, rc);
3815
3816	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3817	}
3818	return rc;
3819	}
3820
3821
3822	/**
3823	* Loads the guest debug registers into the guest-state area in the VMCS.
3824	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3825	*
3826	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3827	*
3828	* @returns VBox status code.
3829	* @param pVCpu Pointer to the VMCPU.
3830	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3831	* out-of-sync. Make sure to update the required fields
3832	* before using them.
3833	*
3834	* @remarks No-long-jump zone!!!
3835	*/
3836	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3837	{
3838	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3839	return VINF_SUCCESS;
3840
3841	#ifdef VBOX_STRICT
3842	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3843	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3844	{
3845	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3846	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3847	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3848	}
3849	#endif
3850
3851	int rc;
3852	PVM pVM = pVCpu->CTX_SUFF(pVM);
3853	bool fInterceptDB = false;
3854	bool fInterceptMovDRx = false;
3855	if ( pVCpu->hm.s.fSingleInstruction
3856	\|\| DBGFIsStepping(pVCpu))
3857	{
3858	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3859	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3860	{
3861	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3862	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3863	AssertRCReturn(rc, rc);
3864	Assert(fInterceptDB == false);
3865	}
3866	else
3867	{
3868	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3869	pVCpu->hm.s.fClearTrapFlag = true;
3870	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3871	fInterceptDB = true;
3872	}
3873	}
3874
3875	if ( fInterceptDB
3876	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3877	{
3878	/*
3879	* Use the combined guest and host DRx values found in the hypervisor
3880	* register set because the debugger has breakpoints active or someone
3881	* is single stepping on the host side without a monitor trap flag.
3882	*
3883	* Note! DBGF expects a clean DR6 state before executing guest code.
3884	*/
3885	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3886	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3887	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3888	{
3889	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3890	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3891	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3892	}
3893	else
3894	#endif
3895	if (!CPUMIsHyperDebugStateActive(pVCpu))
3896	{
3897	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3898	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3899	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3900	}
3901
3902	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3903	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3904	AssertRCReturn(rc, rc);
3905
3906	pVCpu->hm.s.fUsingHyperDR7 = true;
3907	fInterceptDB = true;
3908	fInterceptMovDRx = true;
3909	}
3910	else
3911	{
3912	/*
3913	* If the guest has enabled debug registers, we need to load them prior to
3914	* executing guest code so they'll trigger at the right time.
3915	*/
3916	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3917	{
3918	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3919	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3920	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3921	{
3922	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3923	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3924	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3925	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3926	}
3927	else
3928	#endif
3929	if (!CPUMIsGuestDebugStateActive(pVCpu))
3930	{
3931	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3932	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3933	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3934	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3935	}
3936	Assert(!fInterceptDB);
3937	Assert(!fInterceptMovDRx);
3938	}
3939	/*
3940	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3941	* must intercept #DB in order to maintain a correct DR6 guest value.
3942	*/
3943	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3944	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3945	&& !CPUMIsGuestDebugStateActive(pVCpu))
3946	#else
3947	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3948	#endif
3949	{
3950	fInterceptMovDRx = true;
3951	fInterceptDB = true;
3952	}
3953
3954	/* Update guest DR7. */
3955	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3956	AssertRCReturn(rc, rc);
3957
3958	pVCpu->hm.s.fUsingHyperDR7 = false;
3959	}
3960
3961	/*
3962	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3963	*/
3964	if (fInterceptDB)
3965	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3966	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3967	{
3968	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3969	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3970	#endif
3971	}
3972	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3973	AssertRCReturn(rc, rc);
3974
3975	/*
3976	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3977	*/
3978	if (fInterceptMovDRx)
3979	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3980	else
3981	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3982	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3983	AssertRCReturn(rc, rc);
3984
3985	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3986	return VINF_SUCCESS;
3987	}
3988
3989
3990	#ifdef VBOX_STRICT
3991	/**
3992	* Strict function to validate segment registers.
3993	*
3994	* @remarks ASSUMES CR0 is up to date.
3995	*/
3996	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3997	{
3998	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3999	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4000	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4001	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4002	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4003	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4004	{
4005	/* Protected mode checks */
4006	/* CS */
4007	Assert(pCtx->cs.Attr.n.u1Present);
4008	Assert(!(pCtx->cs.Attr.u & 0xf00));
4009	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4010	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4011	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4012	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4013	\|\| (pCtx->cs.Attr.n.u1Granularity));
4014	/* CS cannot be loaded with NULL in protected mode. */
4015	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4016	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4017	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4018	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4019	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4020	else
4021	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4022	/* SS */
4023	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4024	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4025	if ( !(pCtx->cr0 & X86_CR0_PE)
4026	\|\| pCtx->cs.Attr.n.u4Type == 3)
4027	{
4028	Assert(!pCtx->ss.Attr.n.u2Dpl);
4029	}
4030	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4031	{
4032	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4033	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4034	Assert(pCtx->ss.Attr.n.u1Present);
4035	Assert(!(pCtx->ss.Attr.u & 0xf00));
4036	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4037	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4038	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4039	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4040	\|\| (pCtx->ss.Attr.n.u1Granularity));
4041	}
4042	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4043	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4044	{
4045	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4046	Assert(pCtx->ds.Attr.n.u1Present);
4047	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4048	Assert(!(pCtx->ds.Attr.u & 0xf00));
4049	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4050	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4051	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4052	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4053	\|\| (pCtx->ds.Attr.n.u1Granularity));
4054	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4055	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4056	}
4057	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4058	{
4059	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4060	Assert(pCtx->es.Attr.n.u1Present);
4061	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4062	Assert(!(pCtx->es.Attr.u & 0xf00));
4063	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4064	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4065	\|\| !(pCtx->es.Attr.n.u1Granularity));
4066	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4067	\|\| (pCtx->es.Attr.n.u1Granularity));
4068	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4069	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4070	}
4071	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4072	{
4073	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4074	Assert(pCtx->fs.Attr.n.u1Present);
4075	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4076	Assert(!(pCtx->fs.Attr.u & 0xf00));
4077	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4078	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4079	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4080	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4081	\|\| (pCtx->fs.Attr.n.u1Granularity));
4082	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4083	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4084	}
4085	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4086	{
4087	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4088	Assert(pCtx->gs.Attr.n.u1Present);
4089	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4090	Assert(!(pCtx->gs.Attr.u & 0xf00));
4091	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4092	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4093	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4094	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4095	\|\| (pCtx->gs.Attr.n.u1Granularity));
4096	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4097	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4098	}
4099	/* 64-bit capable CPUs. */
4100	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4101	Assert(!(pCtx->cs.u64Base >> 32));
4102	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4103	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4104	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4105	# endif
4106	}
4107	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4108	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4109	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4110	{
4111	/* Real and v86 mode checks. */
4112	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4113	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4114	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4115	{
4116	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4117	}
4118	else
4119	{
4120	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4121	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4122	}
4123
4124	/* CS */
4125	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4126	Assert(pCtx->cs.u32Limit == 0xffff);
4127	Assert(u32CSAttr == 0xf3);
4128	/* SS */
4129	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4130	Assert(pCtx->ss.u32Limit == 0xffff);
4131	Assert(u32SSAttr == 0xf3);
4132	/* DS */
4133	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4134	Assert(pCtx->ds.u32Limit == 0xffff);
4135	Assert(u32DSAttr == 0xf3);
4136	/* ES */
4137	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4138	Assert(pCtx->es.u32Limit == 0xffff);
4139	Assert(u32ESAttr == 0xf3);
4140	/* FS */
4141	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4142	Assert(pCtx->fs.u32Limit == 0xffff);
4143	Assert(u32FSAttr == 0xf3);
4144	/* GS */
4145	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4146	Assert(pCtx->gs.u32Limit == 0xffff);
4147	Assert(u32GSAttr == 0xf3);
4148	/* 64-bit capable CPUs. */
4149	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4150	Assert(!(pCtx->cs.u64Base >> 32));
4151	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4152	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4153	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4154	# endif
4155	}
4156	}
4157	#endif /* VBOX_STRICT */
4158
4159
4160	/**
4161	* Writes a guest segment register into the guest-state area in the VMCS.
4162	*
4163	* @returns VBox status code.
4164	* @param pVCpu Pointer to the VMCPU.
4165	* @param idxSel Index of the selector in the VMCS.
4166	* @param idxLimit Index of the segment limit in the VMCS.
4167	* @param idxBase Index of the segment base in the VMCS.
4168	* @param idxAccess Index of the access rights of the segment in the VMCS.
4169	* @param pSelReg Pointer to the segment selector.
4170	*
4171	* @remarks No-long-jump zone!!!
4172	*/
4173	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4174	uint32_t idxAccess, PCPUMSELREG pSelReg)
4175	{
4176	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4177	AssertRCReturn(rc, rc);
4178	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4179	AssertRCReturn(rc, rc);
4180	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4181	AssertRCReturn(rc, rc);
4182
4183	uint32_t u32Access = pSelReg->Attr.u;
4184	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4185	{
4186	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4187	u32Access = 0xf3;
4188	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4189	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4190	}
4191	else
4192	{
4193	/*
4194	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4195	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4196	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4197	* loaded in protected-mode have their attribute as 0.
4198	*/
4199	if (!u32Access)
4200	u32Access = X86DESCATTR_UNUSABLE;
4201	}
4202
4203	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4204	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4205	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4206
4207	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4208	AssertRCReturn(rc, rc);
4209	return rc;
4210	}
4211
4212
4213	/**
4214	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4215	* into the guest-state area in the VMCS.
4216	*
4217	* @returns VBox status code.
4218	* @param pVM Pointer to the VM.
4219	* @param pVCPU Pointer to the VMCPU.
4220	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4221	* out-of-sync. Make sure to update the required fields
4222	* before using them.
4223	*
4224	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4225	* @remarks No-long-jump zone!!!
4226	*/
4227	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4228	{
4229	int rc = VERR_INTERNAL_ERROR_5;
4230	PVM pVM = pVCpu->CTX_SUFF(pVM);
4231
4232	/*
4233	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4234	*/
4235	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4236	{
4237	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4238	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4239	{
4240	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4241	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4242	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4243	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4244	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4245	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4246	}
4247
4248	#ifdef VBOX_WITH_REM
4249	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4250	{
4251	Assert(pVM->hm.s.vmx.pRealModeTSS);
4252	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4253	if ( pVCpu->hm.s.vmx.fWasInRealMode
4254	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4255	{
4256	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4257	in real-mode (e.g. OpenBSD 4.0) */
4258	REMFlushTBs(pVM);
4259	Log4(("Load: Switch to protected mode detected!\n"));
4260	pVCpu->hm.s.vmx.fWasInRealMode = false;
4261	}
4262	}
4263	#endif
4264	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4265	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4266	AssertRCReturn(rc, rc);
4267	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4268	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4269	AssertRCReturn(rc, rc);
4270	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4271	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4272	AssertRCReturn(rc, rc);
4273	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4274	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4275	AssertRCReturn(rc, rc);
4276	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4277	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4278	AssertRCReturn(rc, rc);
4279	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4280	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4281	AssertRCReturn(rc, rc);
4282
4283	#ifdef VBOX_STRICT
4284	/* Validate. */
4285	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4286	#endif
4287
4288	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4289	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4290	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4291	}
4292
4293	/*
4294	* Guest TR.
4295	*/
4296	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4297	{
4298	/*
4299	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4300	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4301	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4302	*/
4303	uint16_t u16Sel = 0;
4304	uint32_t u32Limit = 0;
4305	uint64_t u64Base = 0;
4306	uint32_t u32AccessRights = 0;
4307
4308	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4309	{
4310	u16Sel = pMixedCtx->tr.Sel;
4311	u32Limit = pMixedCtx->tr.u32Limit;
4312	u64Base = pMixedCtx->tr.u64Base;
4313	u32AccessRights = pMixedCtx->tr.Attr.u;
4314	}
4315	else
4316	{
4317	Assert(pVM->hm.s.vmx.pRealModeTSS);
4318	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4319
4320	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4321	RTGCPHYS GCPhys;
4322	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4323	AssertRCReturn(rc, rc);
4324
4325	X86DESCATTR DescAttr;
4326	DescAttr.u = 0;
4327	DescAttr.n.u1Present = 1;
4328	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4329
4330	u16Sel = 0;
4331	u32Limit = HM_VTX_TSS_SIZE;
4332	u64Base = GCPhys; /* in real-mode phys = virt. */
4333	u32AccessRights = DescAttr.u;
4334	}
4335
4336	/* Validate. */
4337	Assert(!(u16Sel & RT_BIT(2)));
4338	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4339	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4340	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4341	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4342	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4343	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4344	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4345	Assert( (u32Limit & 0xfff) == 0xfff
4346	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4347	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4348	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4349
4350	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4351	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4352	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4353	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4354
4355	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4356	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4357	}
4358
4359	/*
4360	* Guest GDTR.
4361	*/
4362	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4363	{
4364	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4365	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4366
4367	/* Validate. */
4368	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4369
4370	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4371	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4372	}
4373
4374	/*
4375	* Guest LDTR.
4376	*/
4377	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4378	{
4379	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4380	uint32_t u32Access = 0;
4381	if (!pMixedCtx->ldtr.Attr.u)
4382	u32Access = X86DESCATTR_UNUSABLE;
4383	else
4384	u32Access = pMixedCtx->ldtr.Attr.u;
4385
4386	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4387	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4388	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4389	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4390
4391	/* Validate. */
4392	if (!(u32Access & X86DESCATTR_UNUSABLE))
4393	{
4394	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4395	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4396	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4397	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4398	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4399	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4400	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4401	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4402	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4403	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4404	}
4405
4406	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4407	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4408	}
4409
4410	/*
4411	* Guest IDTR.
4412	*/
4413	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4414	{
4415	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4416	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4417
4418	/* Validate. */
4419	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4420
4421	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4422	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4423	}
4424
4425	return VINF_SUCCESS;
4426	}
4427
4428
4429	/**
4430	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4431	* areas. These MSRs will automatically be loaded to the host CPU on every
4432	* successful VM entry and stored from the host CPU on every successful VM exit.
4433	*
4434	* This also creates/updates MSR slots for the host MSRs. The actual host
4435	* MSR values are -not- updated here for performance reasons. See
4436	* hmR0VmxSaveHostMsrs().
4437	*
4438	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4439	*
4440	* @returns VBox status code.
4441	* @param pVCpu Pointer to the VMCPU.
4442	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4443	* out-of-sync. Make sure to update the required fields
4444	* before using them.
4445	*
4446	* @remarks No-long-jump zone!!!
4447	*/
4448	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4449	{
4450	AssertPtr(pVCpu);
4451	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4452
4453	/*
4454	* MSRs that we use the auto-load/store MSR area in the VMCS.
4455	*/
4456	PVM pVM = pVCpu->CTX_SUFF(pVM);
4457	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4458	{
4459	if (pVM->hm.s.fAllow64BitGuests)
4460	{
4461	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4462	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4463	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4464	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4465	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4466	# ifdef DEBUG
4467	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4468	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4469	Log4(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4470	# endif
4471	#endif
4472	}
4473	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4474	}
4475
4476	/*
4477	* Guest Sysenter MSRs.
4478	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4479	* VM-exits on WRMSRs for these MSRs.
4480	*/
4481	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4482	{
4483	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4484	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4485	}
4486
4487	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4488	{
4489	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4490	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4491	}
4492
4493	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4494	{
4495	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4496	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4497	}
4498
4499	return VINF_SUCCESS;
4500	}
4501
4502
4503	/**
4504	* Loads the guest activity state into the guest-state area in the VMCS.
4505	*
4506	* @returns VBox status code.
4507	* @param pVCpu Pointer to the VMCPU.
4508	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4509	* out-of-sync. Make sure to update the required fields
4510	* before using them.
4511	*
4512	* @remarks No-long-jump zone!!!
4513	*/
4514	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4515	{
4516	NOREF(pCtx);
4517	/** @todo See if we can make use of other states, e.g.
4518	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4519	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4520	{
4521	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4522	AssertRCReturn(rc, rc);
4523
4524	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4525	}
4526	return VINF_SUCCESS;
4527	}
4528
4529
4530	/**
4531	* Sets up the appropriate function to run guest code.
4532	*
4533	* @returns VBox status code.
4534	* @param pVCpu Pointer to the VMCPU.
4535	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4536	* out-of-sync. Make sure to update the required fields
4537	* before using them.
4538	*
4539	* @remarks No-long-jump zone!!!
4540	*/
4541	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4542	{
4543	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4544	{
4545	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4546	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4547	#endif
4548	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4549	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4550	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4551	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4552	{
4553	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4554	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4555	}
4556	#else
4557	/* 64-bit host or hybrid host. */
4558	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4559	#endif
4560	}
4561	else
4562	{
4563	/* Guest is not in long mode, use the 32-bit handler. */
4564	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4565	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4566	{
4567	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4568	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4569	}
4570	#else
4571	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4572	#endif
4573	}
4574	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4575	return VINF_SUCCESS;
4576	}
4577
4578
4579	/**
4580	* Wrapper for running the guest code in VT-x.
4581	*
4582	* @returns VBox strict status code.
4583	* @param pVM Pointer to the VM.
4584	* @param pVCpu Pointer to the VMCPU.
4585	* @param pCtx Pointer to the guest-CPU context.
4586	*
4587	* @remarks No-long-jump zone!!!
4588	*/
4589	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4590	{
4591	/*
4592	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4593	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4594	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4595	*/
4596	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4597	/** @todo Add stats for resume vs launch. */
4598	#ifdef VBOX_WITH_KERNEL_USING_XMM
4599	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4600	#else
4601	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4602	#endif
4603	}
4604
4605
4606	/**
4607	* Reports world-switch error and dumps some useful debug info.
4608	*
4609	* @param pVM Pointer to the VM.
4610	* @param pVCpu Pointer to the VMCPU.
4611	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4612	* @param pCtx Pointer to the guest-CPU context.
4613	* @param pVmxTransient Pointer to the VMX transient structure (only
4614	* exitReason updated).
4615	*/
4616	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4617	{
4618	Assert(pVM);
4619	Assert(pVCpu);
4620	Assert(pCtx);
4621	Assert(pVmxTransient);
4622	HMVMX_ASSERT_PREEMPT_SAFE();
4623
4624	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4625	switch (rcVMRun)
4626	{
4627	case VERR_VMX_INVALID_VMXON_PTR:
4628	AssertFailed();
4629	break;
4630	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4631	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4632	{
4633	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4634	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4635	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4636	AssertRC(rc);
4637
4638	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4639	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4640	Cannot do it here as we may have been long preempted. */
4641
4642	#ifdef VBOX_STRICT
4643	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4644	pVmxTransient->uExitReason));
4645	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4646	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4647	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4648	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4649	else
4650	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4651	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4652	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4653
4654	/* VMX control bits. */
4655	uint32_t u32Val;
4656	uint64_t u64Val;
4657	HMVMXHCUINTREG uHCReg;
4658	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4659	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4660	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4661	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4663	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4664	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4665	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4667	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4668	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4669	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4671	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4672	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4673	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4675	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4677	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4678	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4679	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4680	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4681	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4683	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4685	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4686	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4687	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4688	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4689	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4690	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4691	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4692	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4693	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4694	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4695	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4696	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4697	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4698	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4699	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4700
4701	/* Guest bits. */
4702	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4703	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4704	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4705	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4706	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4707	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4708	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4709	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4710
4711	/* Host bits. */
4712	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4713	Log4(("Host CR0 %#RHr\n", uHCReg));
4714	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4715	Log4(("Host CR3 %#RHr\n", uHCReg));
4716	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4717	Log4(("Host CR4 %#RHr\n", uHCReg));
4718
4719	RTGDTR HostGdtr;
4720	PCX86DESCHC pDesc;
4721	ASMGetGDTR(&HostGdtr);
4722	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4723	Log4(("Host CS %#08x\n", u32Val));
4724	if (u32Val < HostGdtr.cbGdt)
4725	{
4726	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4727	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4728	}
4729
4730	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4731	Log4(("Host DS %#08x\n", u32Val));
4732	if (u32Val < HostGdtr.cbGdt)
4733	{
4734	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4735	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4736	}
4737
4738	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4739	Log4(("Host ES %#08x\n", u32Val));
4740	if (u32Val < HostGdtr.cbGdt)
4741	{
4742	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4743	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4744	}
4745
4746	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4747	Log4(("Host FS %#08x\n", u32Val));
4748	if (u32Val < HostGdtr.cbGdt)
4749	{
4750	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4751	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4752	}
4753
4754	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4755	Log4(("Host GS %#08x\n", u32Val));
4756	if (u32Val < HostGdtr.cbGdt)
4757	{
4758	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4759	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4760	}
4761
4762	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4763	Log4(("Host SS %#08x\n", u32Val));
4764	if (u32Val < HostGdtr.cbGdt)
4765	{
4766	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4767	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4768	}
4769
4770	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4771	Log4(("Host TR %#08x\n", u32Val));
4772	if (u32Val < HostGdtr.cbGdt)
4773	{
4774	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4775	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4776	}
4777
4778	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4779	Log4(("Host TR Base %#RHv\n", uHCReg));
4780	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4781	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4782	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4783	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4784	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4785	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4786	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4787	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4788	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4789	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4790	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4791	Log4(("Host RSP %#RHv\n", uHCReg));
4792	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4793	Log4(("Host RIP %#RHv\n", uHCReg));
4794	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4795	if (HMVMX_IS_64BIT_HOST_MODE())
4796	{
4797	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4798	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4799	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4800	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4801	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4802	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4803	}
4804	# endif
4805	#endif /* VBOX_STRICT */
4806	break;
4807	}
4808
4809	default:
4810	/* Impossible */
4811	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4812	break;
4813	}
4814	NOREF(pVM); NOREF(pCtx);
4815	}
4816
4817
4818	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4819	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4820	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4821	#endif
4822	#ifdef VBOX_STRICT
4823	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4824	{
4825	switch (idxField)
4826	{
4827	case VMX_VMCS_GUEST_RIP:
4828	case VMX_VMCS_GUEST_RSP:
4829	case VMX_VMCS_GUEST_SYSENTER_EIP:
4830	case VMX_VMCS_GUEST_SYSENTER_ESP:
4831	case VMX_VMCS_GUEST_GDTR_BASE:
4832	case VMX_VMCS_GUEST_IDTR_BASE:
4833	case VMX_VMCS_GUEST_CS_BASE:
4834	case VMX_VMCS_GUEST_DS_BASE:
4835	case VMX_VMCS_GUEST_ES_BASE:
4836	case VMX_VMCS_GUEST_FS_BASE:
4837	case VMX_VMCS_GUEST_GS_BASE:
4838	case VMX_VMCS_GUEST_SS_BASE:
4839	case VMX_VMCS_GUEST_LDTR_BASE:
4840	case VMX_VMCS_GUEST_TR_BASE:
4841	case VMX_VMCS_GUEST_CR3:
4842	return true;
4843	}
4844	return false;
4845	}
4846
4847	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4848	{
4849	switch (idxField)
4850	{
4851	/* Read-only fields. */
4852	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4853	return true;
4854	}
4855	/* Remaining readable fields should also be writable. */
4856	return hmR0VmxIsValidWriteField(idxField);
4857	}
4858	#endif /* VBOX_STRICT */
4859
4860
4861	/**
4862	* Executes the specified handler in 64-bit mode.
4863	*
4864	* @returns VBox status code.
4865	* @param pVM Pointer to the VM.
4866	* @param pVCpu Pointer to the VMCPU.
4867	* @param pCtx Pointer to the guest CPU context.
4868	* @param enmOp The operation to perform.
4869	* @param cbParam Number of parameters.
4870	* @param paParam Array of 32-bit parameters.
4871	*/
4872	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4873	uint32_t *paParam)
4874	{
4875	int rc, rc2;
4876	PHMGLOBALCPUINFO pCpu;
4877	RTHCPHYS HCPhysCpuPage;
4878	RTCCUINTREG uOldEflags;
4879
4880	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4881	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4882	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4883	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4884
4885	#ifdef VBOX_STRICT
4886	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4887	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4888
4889	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4890	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4891	#endif
4892
4893	/* Disable interrupts. */
4894	uOldEflags = ASMIntDisableFlags();
4895
4896	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4897	RTCPUID idHostCpu = RTMpCpuId();
4898	CPUMR0SetLApic(pVCpu, idHostCpu);
4899	#endif
4900
4901	pCpu = HMR0GetCurrentCpu();
4902	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4903
4904	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4905	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4906
4907	/* Leave VMX Root Mode. */
4908	VMXDisable();
4909
4910	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4911
4912	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4913	CPUMSetHyperEIP(pVCpu, enmOp);
4914	for (int i = (int)cbParam - 1; i >= 0; i--)
4915	CPUMPushHyper(pVCpu, paParam[i]);
4916
4917	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4918
4919	/* Call the switcher. */
4920	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4921	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4922
4923	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4924	/* Make sure the VMX instructions don't cause #UD faults. */
4925	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4926
4927	/* Re-enter VMX Root Mode */
4928	rc2 = VMXEnable(HCPhysCpuPage);
4929	if (RT_FAILURE(rc2))
4930	{
4931	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4932	ASMSetFlags(uOldEflags);
4933	return rc2;
4934	}
4935
4936	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4937	AssertRC(rc2);
4938	Assert(!(ASMGetFlags() & X86_EFL_IF));
4939	ASMSetFlags(uOldEflags);
4940	return rc;
4941	}
4942
4943
4944	/**
4945	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4946	* supporting 64-bit guests.
4947	*
4948	* @returns VBox status code.
4949	* @param fResume Whether to VMLAUNCH or VMRESUME.
4950	* @param pCtx Pointer to the guest-CPU context.
4951	* @param pCache Pointer to the VMCS cache.
4952	* @param pVM Pointer to the VM.
4953	* @param pVCpu Pointer to the VMCPU.
4954	*/
4955	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4956	{
4957	uint32_t aParam[6];
4958	PHMGLOBALCPUINFO pCpu = NULL;
4959	RTHCPHYS HCPhysCpuPage = 0;
4960	int rc = VERR_INTERNAL_ERROR_5;
4961
4962	pCpu = HMR0GetCurrentCpu();
4963	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4964
4965	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4966	pCache->uPos = 1;
4967	pCache->interPD = PGMGetInterPaeCR3(pVM);
4968	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4969	#endif
4970
4971	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4972	pCache->TestIn.HCPhysCpuPage = 0;
4973	pCache->TestIn.HCPhysVmcs = 0;
4974	pCache->TestIn.pCache = 0;
4975	pCache->TestOut.HCPhysVmcs = 0;
4976	pCache->TestOut.pCache = 0;
4977	pCache->TestOut.pCtx = 0;
4978	pCache->TestOut.eflags = 0;
4979	#endif
4980
4981	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4982	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4983	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4984	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4985	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4986	aParam[5] = 0;
4987
4988	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4989	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4990	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4991	#endif
4992	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4993
4994	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4995	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4996	Assert(pCtx->dr[4] == 10);
4997	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4998	#endif
4999
5000	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5001	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5002	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5003	pVCpu->hm.s.vmx.HCPhysVmcs));
5004	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5005	pCache->TestOut.HCPhysVmcs));
5006	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5007	pCache->TestOut.pCache));
5008	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5009	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5010	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5011	pCache->TestOut.pCtx));
5012	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5013	#endif
5014	return rc;
5015	}
5016
5017
5018	/**
5019	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5020	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5021	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5022	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5023	*
5024	* @returns VBox status code.
5025	* @param pVM Pointer to the VM.
5026	* @param pVCpu Pointer to the VMCPU.
5027	*/
5028	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5029	{
5030	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5031	{ \
5032	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5033	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5034	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5035	++cReadFields; \
5036	}
5037
5038	AssertPtr(pVM);
5039	AssertPtr(pVCpu);
5040	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5041	uint32_t cReadFields = 0;
5042
5043	/*
5044	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5045	* and serve to indicate exceptions to the rules.
5046	*/
5047
5048	/* Guest-natural selector base fields. */
5049	#if 0
5050	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5051	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5052	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5053	#endif
5054	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5055	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5056	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5057	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5058	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5059	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5062	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5063	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5064	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5066	#if 0
5067	/* Unused natural width guest-state fields. */
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5069	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5070	#endif
5071	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5072	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5073
5074	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5075	#if 0
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5081	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5082	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5083	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5084	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5085	#endif
5086
5087	/* Natural width guest-state fields. */
5088	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5089	#if 0
5090	/* Currently unused field. */
5091	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5092	#endif
5093
5094	if (pVM->hm.s.fNestedPaging)
5095	{
5096	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5097	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5098	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5099	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5100	}
5101	else
5102	{
5103	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5104	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5105	}
5106
5107	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5108	return VINF_SUCCESS;
5109	}
5110
5111
5112	/**
5113	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5114	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5115	* darwin, running 64-bit guests).
5116	*
5117	* @returns VBox status code.
5118	* @param pVCpu Pointer to the VMCPU.
5119	* @param idxField The VMCS field encoding.
5120	* @param u64Val 16, 32 or 64-bit value.
5121	*/
5122	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5123	{
5124	int rc;
5125	switch (idxField)
5126	{
5127	/*
5128	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5129	*/
5130	/* 64-bit Control fields. */
5131	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5132	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5133	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5134	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5135	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5136	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5137	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5138	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5139	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5140	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5141	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5142	case VMX_VMCS64_CTRL_EPTP_FULL:
5143	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5144	/* 64-bit Guest-state fields. */
5145	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5146	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5147	case VMX_VMCS64_GUEST_PAT_FULL:
5148	case VMX_VMCS64_GUEST_EFER_FULL:
5149	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5150	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5151	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5152	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5153	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5154	/* 64-bit Host-state fields. */
5155	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5156	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5157	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5158	{
5159	rc = VMXWriteVmcs32(idxField, u64Val);
5160	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5161	break;
5162	}
5163
5164	/*
5165	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5166	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5167	*/
5168	/* Natural-width Guest-state fields. */
5169	case VMX_VMCS_GUEST_CR3:
5170	case VMX_VMCS_GUEST_ES_BASE:
5171	case VMX_VMCS_GUEST_CS_BASE:
5172	case VMX_VMCS_GUEST_SS_BASE:
5173	case VMX_VMCS_GUEST_DS_BASE:
5174	case VMX_VMCS_GUEST_FS_BASE:
5175	case VMX_VMCS_GUEST_GS_BASE:
5176	case VMX_VMCS_GUEST_LDTR_BASE:
5177	case VMX_VMCS_GUEST_TR_BASE:
5178	case VMX_VMCS_GUEST_GDTR_BASE:
5179	case VMX_VMCS_GUEST_IDTR_BASE:
5180	case VMX_VMCS_GUEST_RSP:
5181	case VMX_VMCS_GUEST_RIP:
5182	case VMX_VMCS_GUEST_SYSENTER_ESP:
5183	case VMX_VMCS_GUEST_SYSENTER_EIP:
5184	{
5185	if (!(u64Val >> 32))
5186	{
5187	/* If this field is 64-bit, VT-x will zero out the top bits. */
5188	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5189	}
5190	else
5191	{
5192	/* Assert that only the 32->64 switcher case should ever come here. */
5193	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5194	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5195	}
5196	break;
5197	}
5198
5199	default:
5200	{
5201	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5202	rc = VERR_INVALID_PARAMETER;
5203	break;
5204	}
5205	}
5206	AssertRCReturn(rc, rc);
5207	return rc;
5208	}
5209
5210
5211	/**
5212	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5213	* hosts (except darwin) for 64-bit guests.
5214	*
5215	* @param pVCpu Pointer to the VMCPU.
5216	* @param idxField The VMCS field encoding.
5217	* @param u64Val 16, 32 or 64-bit value.
5218	*/
5219	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5220	{
5221	AssertPtr(pVCpu);
5222	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5223
5224	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5225	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5226
5227	/* Make sure there are no duplicates. */
5228	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5229	{
5230	if (pCache->Write.aField[i] == idxField)
5231	{
5232	pCache->Write.aFieldVal[i] = u64Val;
5233	return VINF_SUCCESS;
5234	}
5235	}
5236
5237	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5238	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5239	pCache->Write.cValidEntries++;
5240	return VINF_SUCCESS;
5241	}
5242
5243	/* Enable later when the assembly code uses these as callbacks. */
5244	#if 0
5245	/*
5246	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5247	*
5248	* @param pVCpu Pointer to the VMCPU.
5249	* @param pCache Pointer to the VMCS cache.
5250	*
5251	* @remarks No-long-jump zone!!!
5252	*/
5253	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5254	{
5255	AssertPtr(pCache);
5256	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5257	{
5258	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5259	AssertRC(rc);
5260	}
5261	pCache->Write.cValidEntries = 0;
5262	}
5263
5264
5265	/**
5266	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5267	*
5268	* @param pVCpu Pointer to the VMCPU.
5269	* @param pCache Pointer to the VMCS cache.
5270	*
5271	* @remarks No-long-jump zone!!!
5272	*/
5273	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5274	{
5275	AssertPtr(pCache);
5276	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5277	{
5278	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5279	AssertRC(rc);
5280	}
5281	}
5282	#endif
5283	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5284
5285
5286	/**
5287	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5288	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5289	* timer.
5290	*
5291	* @returns VBox status code.
5292	* @param pVCpu Pointer to the VMCPU.
5293	*
5294	* @remarks No-long-jump zone!!!
5295	*/
5296	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5297	{
5298	int rc = VERR_INTERNAL_ERROR_5;
5299	bool fOffsettedTsc = false;
5300	PVM pVM = pVCpu->CTX_SUFF(pVM);
5301	if (pVM->hm.s.vmx.fUsePreemptTimer)
5302	{
5303	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5304
5305	/* Make sure the returned values have sane upper and lower boundaries. */
5306	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5307	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5308	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5309	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5310
5311	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5312	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5313	}
5314	else
5315	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5316
5317	if (fOffsettedTsc)
5318	{
5319	uint64_t u64CurTSC = ASMReadTSC();
5320	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5321	{
5322	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5323	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5324
5325	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5326	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5327	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5328	}
5329	else
5330	{
5331	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5332	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5333	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5334	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5335	}
5336	}
5337	else
5338	{
5339	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5340	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5341	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5342	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5343	}
5344	}
5345
5346
5347	/**
5348	* Determines if an exception is a contributory exception. Contributory
5349	* exceptions are ones which can cause double-faults. Page-fault is
5350	* intentionally not included here as it's a conditional contributory exception.
5351	*
5352	* @returns true if the exception is contributory, false otherwise.
5353	* @param uVector The exception vector.
5354	*/
5355	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5356	{
5357	switch (uVector)
5358	{
5359	case X86_XCPT_GP:
5360	case X86_XCPT_SS:
5361	case X86_XCPT_NP:
5362	case X86_XCPT_TS:
5363	case X86_XCPT_DE:
5364	return true;
5365	default:
5366	break;
5367	}
5368	return false;
5369	}
5370
5371
5372	/**
5373	* Sets an event as a pending event to be injected into the guest.
5374	*
5375	* @param pVCpu Pointer to the VMCPU.
5376	* @param u32IntInfo The VM-entry interruption-information field.
5377	* @param cbInstr The VM-entry instruction length in bytes (for software
5378	* interrupts, exceptions and privileged software
5379	* exceptions).
5380	* @param u32ErrCode The VM-entry exception error code.
5381	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5382	* page-fault.
5383	*
5384	* @remarks Statistics counter assumes this is a guest event being injected or
5385	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5386	* always incremented.
5387	*/
5388	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5389	RTGCUINTPTR GCPtrFaultAddress)
5390	{
5391	Assert(!pVCpu->hm.s.Event.fPending);
5392	pVCpu->hm.s.Event.fPending = true;
5393	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5394	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5395	pVCpu->hm.s.Event.cbInstr = cbInstr;
5396	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5397
5398	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5399	}
5400
5401
5402	/**
5403	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5404	*
5405	* @param pVCpu Pointer to the VMCPU.
5406	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5407	* out-of-sync. Make sure to update the required fields
5408	* before using them.
5409	*/
5410	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5411	{
5412	NOREF(pMixedCtx);
5413	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5414	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5415	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5416	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5417	}
5418
5419
5420	/**
5421	* Handle a condition that occurred while delivering an event through the guest
5422	* IDT.
5423	*
5424	* @returns VBox status code (informational error codes included).
5425	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5426	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5427	* continue execution of the guest which will delivery the #DF.
5428	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5429	*
5430	* @param pVCpu Pointer to the VMCPU.
5431	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5432	* out-of-sync. Make sure to update the required fields
5433	* before using them.
5434	* @param pVmxTransient Pointer to the VMX transient structure.
5435	*
5436	* @remarks No-long-jump zone!!!
5437	*/
5438	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5439	{
5440	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5441	AssertRCReturn(rc, rc);
5442	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5443	{
5444	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5445	AssertRCReturn(rc, rc);
5446
5447	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5448	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5449	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5450
5451	typedef enum
5452	{
5453	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5454	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5455	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5456	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5457	} VMXREFLECTXCPT;
5458
5459	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5460	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5461	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5462	{
5463	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5464	{
5465	enmReflect = VMXREFLECTXCPT_XCPT;
5466	#ifdef VBOX_STRICT
5467	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5468	&& uExitVector == X86_XCPT_PF)
5469	{
5470	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5471	}
5472	#endif
5473	if ( uExitVector == X86_XCPT_PF
5474	&& uIdtVector == X86_XCPT_PF)
5475	{
5476	pVmxTransient->fVectoringPF = true;
5477	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5478	}
5479	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5480	&& hmR0VmxIsContributoryXcpt(uExitVector)
5481	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5482	\|\| uIdtVector == X86_XCPT_PF))
5483	{
5484	enmReflect = VMXREFLECTXCPT_DF;
5485	}
5486	else if (uIdtVector == X86_XCPT_DF)
5487	enmReflect = VMXREFLECTXCPT_TF;
5488	}
5489	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5490	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5491	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5492	{
5493	/*
5494	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5495	* (whatever they are) as they reoccur when restarting the instruction.
5496	*/
5497	enmReflect = VMXREFLECTXCPT_XCPT;
5498	}
5499	}
5500	else
5501	{
5502	/*
5503	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5504	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5505	* original exception to the guest after handling the VM-exit.
5506	*/
5507	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5508	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5509	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5510	{
5511	enmReflect = VMXREFLECTXCPT_XCPT;
5512	}
5513	}
5514
5515	switch (enmReflect)
5516	{
5517	case VMXREFLECTXCPT_XCPT:
5518	{
5519	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5520	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5521	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5522
5523	uint32_t u32ErrCode = 0;
5524	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5525	{
5526	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5527	AssertRCReturn(rc, rc);
5528	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5529	}
5530
5531	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5532	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5533	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5534	rc = VINF_SUCCESS;
5535	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5536	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5537
5538	break;
5539	}
5540
5541	case VMXREFLECTXCPT_DF:
5542	{
5543	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5544	rc = VINF_HM_DOUBLE_FAULT;
5545	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5546	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5547
5548	break;
5549	}
5550
5551	case VMXREFLECTXCPT_TF:
5552	{
5553	rc = VINF_EM_RESET;
5554	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5555	uExitVector));
5556	break;
5557	}
5558
5559	default:
5560	Assert(rc == VINF_SUCCESS);
5561	break;
5562	}
5563	}
5564	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5565	return rc;
5566	}
5567
5568
5569	/**
5570	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5571	*
5572	* @returns VBox status code.
5573	* @param pVCpu Pointer to the VMCPU.
5574	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5575	* out-of-sync. Make sure to update the required fields
5576	* before using them.
5577	*
5578	* @remarks No-long-jump zone!!!
5579	*/
5580	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5581	{
5582	NOREF(pMixedCtx);
5583
5584	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5585	{
5586	uint32_t uVal = 0;
5587	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5588	AssertRCReturn(rc, rc);
5589
5590	uint32_t uShadow = 0;
5591	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5592	AssertRCReturn(rc, rc);
5593
5594	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5595	CPUMSetGuestCR0(pVCpu, uVal);
5596	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5597	}
5598	return VINF_SUCCESS;
5599	}
5600
5601
5602	/**
5603	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5604	*
5605	* @returns VBox status code.
5606	* @param pVCpu Pointer to the VMCPU.
5607	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5608	* out-of-sync. Make sure to update the required fields
5609	* before using them.
5610	*
5611	* @remarks No-long-jump zone!!!
5612	*/
5613	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5614	{
5615	NOREF(pMixedCtx);
5616
5617	int rc = VINF_SUCCESS;
5618	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5619	{
5620	uint32_t uVal = 0;
5621	uint32_t uShadow = 0;
5622	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5623	AssertRCReturn(rc, rc);
5624	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5625	AssertRCReturn(rc, rc);
5626
5627	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5628	CPUMSetGuestCR4(pVCpu, uVal);
5629	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5630	}
5631	return rc;
5632	}
5633
5634
5635	/**
5636	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5637	*
5638	* @returns VBox status code.
5639	* @param pVCpu Pointer to the VMCPU.
5640	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5641	* out-of-sync. Make sure to update the required fields
5642	* before using them.
5643	*
5644	* @remarks No-long-jump zone!!!
5645	*/
5646	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5647	{
5648	int rc = VINF_SUCCESS;
5649	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5650	{
5651	uint64_t u64Val = 0;
5652	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5653	AssertRCReturn(rc, rc);
5654
5655	pMixedCtx->rip = u64Val;
5656	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5657	}
5658	return rc;
5659	}
5660
5661
5662	/**
5663	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5664	*
5665	* @returns VBox status code.
5666	* @param pVCpu Pointer to the VMCPU.
5667	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5668	* out-of-sync. Make sure to update the required fields
5669	* before using them.
5670	*
5671	* @remarks No-long-jump zone!!!
5672	*/
5673	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5674	{
5675	int rc = VINF_SUCCESS;
5676	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5677	{
5678	uint64_t u64Val = 0;
5679	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5680	AssertRCReturn(rc, rc);
5681
5682	pMixedCtx->rsp = u64Val;
5683	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5684	}
5685	return rc;
5686	}
5687
5688
5689	/**
5690	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5691	*
5692	* @returns VBox status code.
5693	* @param pVCpu Pointer to the VMCPU.
5694	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5695	* out-of-sync. Make sure to update the required fields
5696	* before using them.
5697	*
5698	* @remarks No-long-jump zone!!!
5699	*/
5700	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5701	{
5702	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5703	{
5704	uint32_t uVal = 0;
5705	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5706	AssertRCReturn(rc, rc);
5707
5708	pMixedCtx->eflags.u32 = uVal;
5709	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5710	{
5711	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5712	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5713
5714	pMixedCtx->eflags.Bits.u1VM = 0;
5715	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5716	}
5717
5718	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5719	}
5720	return VINF_SUCCESS;
5721	}
5722
5723
5724	/**
5725	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5726	* guest-CPU context.
5727	*/
5728	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5729	{
5730	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5731	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5732	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5733	return rc;
5734	}
5735
5736
5737	/**
5738	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5739	* from the guest-state area in the VMCS.
5740	*
5741	* @param pVCpu Pointer to the VMCPU.
5742	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5743	* out-of-sync. Make sure to update the required fields
5744	* before using them.
5745	*
5746	* @remarks No-long-jump zone!!!
5747	*/
5748	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5749	{
5750	uint32_t uIntrState = 0;
5751	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5752	AssertRC(rc);
5753
5754	if (!uIntrState)
5755	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5756	else
5757	{
5758	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5759	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5760	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5761	AssertRC(rc);
5762	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5763	AssertRC(rc);
5764
5765	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5766	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5767	}
5768	}
5769
5770
5771	/**
5772	* Saves the guest's activity state.
5773	*
5774	* @returns VBox status code.
5775	* @param pVCpu Pointer to the VMCPU.
5776	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5777	* out-of-sync. Make sure to update the required fields
5778	* before using them.
5779	*
5780	* @remarks No-long-jump zone!!!
5781	*/
5782	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5783	{
5784	NOREF(pMixedCtx);
5785	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5786	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5787	return VINF_SUCCESS;
5788	}
5789
5790
5791	/**
5792	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5793	* the current VMCS into the guest-CPU context.
5794	*
5795	* @returns VBox status code.
5796	* @param pVCpu Pointer to the VMCPU.
5797	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5798	* out-of-sync. Make sure to update the required fields
5799	* before using them.
5800	*
5801	* @remarks No-long-jump zone!!!
5802	*/
5803	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5804	{
5805	int rc = VINF_SUCCESS;
5806	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5807	{
5808	uint32_t u32Val = 0;
5809	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5810	pMixedCtx->SysEnter.cs = u32Val;
5811	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5812	}
5813
5814	uint64_t u64Val = 0;
5815	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5816	{
5817	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5818	pMixedCtx->SysEnter.eip = u64Val;
5819	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5820	}
5821	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5822	{
5823	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5824	pMixedCtx->SysEnter.esp = u64Val;
5825	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5826	}
5827	return rc;
5828	}
5829
5830
5831	/**
5832	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5833	* the CPU back into the guest-CPU context.
5834	*
5835	* @returns VBox status code.
5836	* @param pVCpu Pointer to the VMCPU.
5837	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5838	* out-of-sync. Make sure to update the required fields
5839	* before using them.
5840	*
5841	* @remarks No-long-jump zone!!!
5842	*/
5843	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5844	{
5845	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS)
5846	return VINF_SUCCESS;
5847
5848	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5849	if ( HMVMX_IS_64BIT_HOST_MODE()
5850	&& pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5851	{
5852	/* We should not get preempted to a different CPU at this point while reading the MSRs. */
5853	VMMRZCallRing3Disable(pVCpu);
5854	HM_DISABLE_PREEMPT_IF_NEEDED();
5855
5856	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5857
5858	HM_RESTORE_PREEMPT_IF_NEEDED();
5859	VMMRZCallRing3Enable(pVCpu);
5860	}
5861	#endif
5862
5863	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5864	return VINF_SUCCESS;
5865	}
5866
5867
5868	/**
5869	* Saves the auto load/store'd guest MSRs from the current VMCS into
5870	* the guest-CPU context.
5871	*
5872	* @returns VBox status code.
5873	* @param pVCpu Pointer to the VMCPU.
5874	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5875	* out-of-sync. Make sure to update the required fields
5876	* before using them.
5877	*
5878	* @remarks No-long-jump zone!!!
5879	*/
5880	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5881	{
5882	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5883	return VINF_SUCCESS;
5884
5885	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5886	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5887	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5888	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5889	{
5890	switch (pMsr->u32Msr)
5891	{
5892	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5893	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5894	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5895	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5896	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5897	default:
5898	{
5899	AssertFailed();
5900	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5901	}
5902	}
5903	}
5904
5905	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5906	return VINF_SUCCESS;
5907	}
5908
5909
5910	/**
5911	* Saves the guest control registers from the current VMCS into the guest-CPU
5912	* context.
5913	*
5914	* @returns VBox status code.
5915	* @param pVCpu Pointer to the VMCPU.
5916	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5917	* out-of-sync. Make sure to update the required fields
5918	* before using them.
5919	*
5920	* @remarks No-long-jump zone!!!
5921	*/
5922	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5923	{
5924	/* Guest CR0. Guest FPU. */
5925	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5926	AssertRCReturn(rc, rc);
5927
5928	/* Guest CR4. */
5929	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5930	AssertRCReturn(rc, rc);
5931
5932	/* Guest CR2 - updated always during the world-switch or in #PF. */
5933	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5934	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5935	{
5936	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5937	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5938
5939	PVM pVM = pVCpu->CTX_SUFF(pVM);
5940	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5941	\|\| ( pVM->hm.s.fNestedPaging
5942	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5943	{
5944	uint64_t u64Val = 0;
5945	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5946	if (pMixedCtx->cr3 != u64Val)
5947	{
5948	CPUMSetGuestCR3(pVCpu, u64Val);
5949	if (VMMRZCallRing3IsEnabled(pVCpu))
5950	{
5951	PGMUpdateCR3(pVCpu, u64Val);
5952	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5953	}
5954	else
5955	{
5956	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5957	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5958	}
5959	}
5960
5961	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5962	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5963	{
5964	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5965	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5966	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5967	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5968
5969	if (VMMRZCallRing3IsEnabled(pVCpu))
5970	{
5971	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5972	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5973	}
5974	else
5975	{
5976	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5977	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5978	}
5979	}
5980	}
5981
5982	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5983	}
5984
5985	/*
5986	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5987	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5988	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5989	*
5990	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5991	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5992	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5993	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5994	*
5995	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5996	*/
5997	if (VMMRZCallRing3IsEnabled(pVCpu))
5998	{
5999	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6000	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6001
6002	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6003	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6004
6005	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6006	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6007	}
6008
6009	return rc;
6010	}
6011
6012
6013	/**
6014	* Reads a guest segment register from the current VMCS into the guest-CPU
6015	* context.
6016	*
6017	* @returns VBox status code.
6018	* @param pVCpu Pointer to the VMCPU.
6019	* @param idxSel Index of the selector in the VMCS.
6020	* @param idxLimit Index of the segment limit in the VMCS.
6021	* @param idxBase Index of the segment base in the VMCS.
6022	* @param idxAccess Index of the access rights of the segment in the VMCS.
6023	* @param pSelReg Pointer to the segment selector.
6024	*
6025	* @remarks No-long-jump zone!!!
6026	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6027	* macro as that takes care of whether to read from the VMCS cache or
6028	* not.
6029	*/
6030	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6031	PCPUMSELREG pSelReg)
6032	{
6033	NOREF(pVCpu);
6034
6035	uint32_t u32Val = 0;
6036	int rc = VMXReadVmcs32(idxSel, &u32Val);
6037	AssertRCReturn(rc, rc);
6038	pSelReg->Sel = (uint16_t)u32Val;
6039	pSelReg->ValidSel = (uint16_t)u32Val;
6040	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6041
6042	rc = VMXReadVmcs32(idxLimit, &u32Val);
6043	AssertRCReturn(rc, rc);
6044	pSelReg->u32Limit = u32Val;
6045
6046	uint64_t u64Val = 0;
6047	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6048	AssertRCReturn(rc, rc);
6049	pSelReg->u64Base = u64Val;
6050
6051	rc = VMXReadVmcs32(idxAccess, &u32Val);
6052	AssertRCReturn(rc, rc);
6053	pSelReg->Attr.u = u32Val;
6054
6055	/*
6056	* If VT-x marks the segment as unusable, most other bits remain undefined:
6057	* - For CS the L, D and G bits have meaning.
6058	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6059	* - For the remaining data segments no bits are defined.
6060	*
6061	* The present bit and the unusable bit has been observed to be set at the
6062	* same time (the selector was supposed to invalid as we started executing
6063	* a V8086 interrupt in ring-0).
6064	*
6065	* What should be important for the rest of the VBox code that the P bit is
6066	* cleared. Some of the other VBox code recognizes the unusable bit, but
6067	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6068	* safe side here, we'll strip off P and other bits we don't care about. If
6069	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6070	*
6071	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6072	*/
6073	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6074	{
6075	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6076
6077	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6078	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6079	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6080
6081	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6082	#ifdef DEBUG_bird
6083	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6084	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6085	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6086	#endif
6087	}
6088	return VINF_SUCCESS;
6089	}
6090
6091
6092	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6093	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6094	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6095	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6096	#else
6097	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6098	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6099	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6100	#endif
6101
6102
6103	/**
6104	* Saves the guest segment registers from the current VMCS into the guest-CPU
6105	* context.
6106	*
6107	* @returns VBox status code.
6108	* @param pVCpu Pointer to the VMCPU.
6109	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6110	* out-of-sync. Make sure to update the required fields
6111	* before using them.
6112	*
6113	* @remarks No-long-jump zone!!!
6114	*/
6115	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6116	{
6117	/* Guest segment registers. */
6118	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6119	{
6120	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6121	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6122	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6123	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6124	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6125	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6126	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6127
6128	/* Restore segment attributes for real-on-v86 mode hack. */
6129	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6130	{
6131	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6132	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6133	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6134	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6135	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6136	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6137	}
6138	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
6139	}
6140
6141	return VINF_SUCCESS;
6142	}
6143
6144
6145	/**
6146	* Saves the guest descriptor table registers and task register from the current
6147	* VMCS into the guest-CPU context.
6148	*
6149	* @returns VBox status code.
6150	* @param pVCpu Pointer to the VMCPU.
6151	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6152	* out-of-sync. Make sure to update the required fields
6153	* before using them.
6154	*
6155	* @remarks No-long-jump zone!!!
6156	*/
6157	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6158	{
6159	int rc = VINF_SUCCESS;
6160
6161	/* Guest LDTR. */
6162	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
6163	{
6164	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6165	AssertRCReturn(rc, rc);
6166	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
6167	}
6168
6169	/* Guest GDTR. */
6170	uint64_t u64Val = 0;
6171	uint32_t u32Val = 0;
6172	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6173	{
6174	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6175	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6176	pMixedCtx->gdtr.pGdt = u64Val;
6177	pMixedCtx->gdtr.cbGdt = u32Val;
6178	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
6179	}
6180
6181	/* Guest IDTR. */
6182	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6183	{
6184	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6185	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6186	pMixedCtx->idtr.pIdt = u64Val;
6187	pMixedCtx->idtr.cbIdt = u32Val;
6188	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
6189	}
6190
6191	/* Guest TR. */
6192	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6193	{
6194	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6195	AssertRCReturn(rc, rc);
6196
6197	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6198	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6199	{
6200	rc = VMXLOCAL_READ_SEG(TR, tr);
6201	AssertRCReturn(rc, rc);
6202	}
6203	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
6204	}
6205	return rc;
6206	}
6207
6208	#undef VMXLOCAL_READ_SEG
6209
6210
6211	/**
6212	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6213	* context.
6214	*
6215	* @returns VBox status code.
6216	* @param pVCpu Pointer to the VMCPU.
6217	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6218	* out-of-sync. Make sure to update the required fields
6219	* before using them.
6220	*
6221	* @remarks No-long-jump zone!!!
6222	*/
6223	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6224	{
6225	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6226	{
6227	if (!pVCpu->hm.s.fUsingHyperDR7)
6228	{
6229	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6230	uint32_t u32Val;
6231	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6232	pMixedCtx->dr[7] = u32Val;
6233	}
6234
6235	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
6236	}
6237	return VINF_SUCCESS;
6238	}
6239
6240
6241	/**
6242	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6243	*
6244	* @returns VBox status code.
6245	* @param pVCpu Pointer to the VMCPU.
6246	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6247	* out-of-sync. Make sure to update the required fields
6248	* before using them.
6249	*
6250	* @remarks No-long-jump zone!!!
6251	*/
6252	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6253	{
6254	NOREF(pMixedCtx);
6255
6256	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6257	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
6258	return VINF_SUCCESS;
6259	}
6260
6261
6262	/**
6263	* Saves the entire guest state from the currently active VMCS into the
6264	* guest-CPU context. This essentially VMREADs all guest-data.
6265	*
6266	* @returns VBox status code.
6267	* @param pVCpu Pointer to the VMCPU.
6268	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6269	* out-of-sync. Make sure to update the required fields
6270	* before using them.
6271	*/
6272	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6273	{
6274	Assert(pVCpu);
6275	Assert(pMixedCtx);
6276
6277	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6278	return VINF_SUCCESS;
6279
6280	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6281	again on the ring-3 callback path, there is no real need to. */
6282	if (VMMRZCallRing3IsEnabled(pVCpu))
6283	VMMR0LogFlushDisable(pVCpu);
6284	else
6285	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6286	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6287
6288	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6289	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6290
6291	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6292	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6293
6294	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6295	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6296
6297	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6298	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6299
6300	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6301	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6302
6303	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6304	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6305
6306	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6308
6309	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6311
6312	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6317
6318	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6319	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6320
6321	if (VMMRZCallRing3IsEnabled(pVCpu))
6322	VMMR0LogFlushEnable(pVCpu);
6323
6324	return rc;
6325	}
6326
6327
6328	/**
6329	* Check per-VM and per-VCPU force flag actions that require us to go back to
6330	* ring-3 for one reason or another.
6331	*
6332	* @returns VBox status code (information status code included).
6333	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6334	* ring-3.
6335	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6336	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6337	* interrupts)
6338	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6339	* all EMTs to be in ring-3.
6340	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6341	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6342	* to the EM loop.
6343	*
6344	* @param pVM Pointer to the VM.
6345	* @param pVCpu Pointer to the VMCPU.
6346	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6347	* out-of-sync. Make sure to update the required fields
6348	* before using them.
6349	*/
6350	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6351	{
6352	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6353
6354	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6355	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6356	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6357	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6358	{
6359	/* We need the control registers now, make sure the guest-CPU context is updated. */
6360	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6361	AssertRCReturn(rc3, rc3);
6362
6363	/* Pending HM CR3 sync. */
6364	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6365	{
6366	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6367	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6368	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6369	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6370	}
6371
6372	/* Pending HM PAE PDPEs. */
6373	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6374	{
6375	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6376	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6377	}
6378
6379	/* Pending PGM C3 sync. */
6380	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6381	{
6382	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6383	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6384	if (rc2 != VINF_SUCCESS)
6385	{
6386	AssertRC(rc2);
6387	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6388	return rc2;
6389	}
6390	}
6391
6392	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6393	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6394	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6395	{
6396	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6397	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6398	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6399	return rc2;
6400	}
6401
6402	/* Pending VM request packets, such as hardware interrupts. */
6403	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6404	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6405	{
6406	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6407	return VINF_EM_PENDING_REQUEST;
6408	}
6409
6410	/* Pending PGM pool flushes. */
6411	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6412	{
6413	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6414	return VINF_PGM_POOL_FLUSH_PENDING;
6415	}
6416
6417	/* Pending DMA requests. */
6418	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6419	{
6420	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6421	return VINF_EM_RAW_TO_R3;
6422	}
6423	}
6424
6425	return VINF_SUCCESS;
6426	}
6427
6428
6429	/**
6430	* Converts any TRPM trap into a pending HM event. This is typically used when
6431	* entering from ring-3 (not longjmp returns).
6432	*
6433	* @param pVCpu Pointer to the VMCPU.
6434	*/
6435	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6436	{
6437	Assert(TRPMHasTrap(pVCpu));
6438	Assert(!pVCpu->hm.s.Event.fPending);
6439
6440	uint8_t uVector;
6441	TRPMEVENT enmTrpmEvent;
6442	RTGCUINT uErrCode;
6443	RTGCUINTPTR GCPtrFaultAddress;
6444	uint8_t cbInstr;
6445
6446	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6447	AssertRC(rc);
6448
6449	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6450	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6451	if (enmTrpmEvent == TRPM_TRAP)
6452	{
6453	switch (uVector)
6454	{
6455	case X86_XCPT_BP:
6456	case X86_XCPT_OF:
6457	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6458	break;
6459
6460	case X86_XCPT_PF:
6461	case X86_XCPT_DF:
6462	case X86_XCPT_TS:
6463	case X86_XCPT_NP:
6464	case X86_XCPT_SS:
6465	case X86_XCPT_GP:
6466	case X86_XCPT_AC:
6467	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6468	/* no break! */
6469	default:
6470	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6471	break;
6472	}
6473	}
6474	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6475	{
6476	if (uVector == X86_XCPT_NMI)
6477	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6478	else
6479	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6480	}
6481	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6482	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6483	else
6484	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6485
6486	rc = TRPMResetTrap(pVCpu);
6487	AssertRC(rc);
6488	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6489	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6490
6491	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6492	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6493	}
6494
6495
6496	/**
6497	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6498	* VT-x to execute any instruction.
6499	*
6500	* @param pvCpu Pointer to the VMCPU.
6501	*/
6502	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6503	{
6504	Assert(pVCpu->hm.s.Event.fPending);
6505
6506	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6507	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6508	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6509	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6510
6511	/* If a trap was already pending, we did something wrong! */
6512	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6513
6514	TRPMEVENT enmTrapType;
6515	switch (uVectorType)
6516	{
6517	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6518	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6519	enmTrapType = TRPM_HARDWARE_INT;
6520	break;
6521
6522	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6523	enmTrapType = TRPM_SOFTWARE_INT;
6524	break;
6525
6526	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6527	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6528	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6529	enmTrapType = TRPM_TRAP;
6530	break;
6531
6532	default:
6533	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6534	enmTrapType = TRPM_32BIT_HACK;
6535	break;
6536	}
6537
6538	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6539
6540	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6541	AssertRC(rc);
6542
6543	if (fErrorCodeValid)
6544	TRPMSetErrorCode(pVCpu, uErrorCode);
6545
6546	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6547	&& uVector == X86_XCPT_PF)
6548	{
6549	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6550	}
6551	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6552	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6553	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6554	{
6555	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6556	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6557	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6558	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6559	}
6560	pVCpu->hm.s.Event.fPending = false;
6561	}
6562
6563
6564	/**
6565	* Does the necessary state syncing before returning to ring-3 for any reason
6566	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6567	*
6568	* @returns VBox status code.
6569	* @param pVM Pointer to the VM.
6570	* @param pVCpu Pointer to the VMCPU.
6571	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6572	* be out-of-sync. Make sure to update the required
6573	* fields before using them.
6574	* @param fSaveGuestState Whether to save the guest state or not.
6575	*
6576	* @remarks If you modify code here, make sure to check whether
6577	* hmR0VmxCallRing3Callback() needs to be updated too!!!
6578	* @remarks No-long-jmp zone!!!
6579	*/
6580	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6581	{
6582	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6583	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6584
6585	RTCPUID idCpu = RTMpCpuId();
6586	Log4Func(("HostCpuId=%u\n", idCpu));
6587
6588	/* Save the guest state if necessary. */
6589	if ( fSaveGuestState
6590	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6591	{
6592	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6593	AssertRCReturn(rc, rc);
6594	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6595	}
6596
6597	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6598	if (CPUMIsGuestFPUStateActive(pVCpu))
6599	{
6600	/* We shouldn't reload CR0 without saving it first. */
6601	if (!fSaveGuestState)
6602	{
6603	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6604	AssertRCReturn(rc, rc);
6605	}
6606	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6607	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6608	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6609	}
6610
6611	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6612	#ifdef VBOX_STRICT
6613	if (CPUMIsHyperDebugStateActive(pVCpu))
6614	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6615	#endif
6616	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6617	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6618	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6619	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6620
6621	#if HC_ARCH_BITS == 64
6622	/* Restore host-state bits that VT-x only restores partially. */
6623	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6624	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6625	{
6626	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6627	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6628	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6629	}
6630	#endif
6631
6632	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6633	/* Restore the host MSRs as we're leaving VT-x context. */
6634	if ( HMVMX_IS_64BIT_HOST_MODE()
6635	&& pVM->hm.s.fAllow64BitGuests
6636	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6637	{
6638	/* We shouldn't reload the guest MSRs without saving it first. */
6639	if (!fSaveGuestState)
6640	{
6641	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6642	AssertRCReturn(rc, rc);
6643	}
6644	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS);
6645	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6646	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6647	}
6648	#endif
6649
6650	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6651	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6652	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6653	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6654	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6655	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6656	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6657	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6658
6659	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6660
6661	/** @todo This partially defeats the purpose of having preemption hooks.
6662	* The problem is, deregistering the hooks should be moved to a place that
6663	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6664	* context.
6665	*/
6666	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6667	{
6668	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6669	AssertRCReturn(rc, rc);
6670
6671	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6672	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6673	}
6674	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6675	NOREF(idCpu);
6676
6677	return VINF_SUCCESS;
6678	}
6679
6680
6681	/**
6682	* Leaves the VT-x session.
6683	*
6684	* @returns VBox status code.
6685	* @param pVM Pointer to the VM.
6686	* @param pVCpu Pointer to the VMCPU.
6687	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6688	* out-of-sync. Make sure to update the required fields
6689	* before using them.
6690	*
6691	* @remarks No-long-jmp zone!!!
6692	*/
6693	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6694	{
6695	HM_DISABLE_PREEMPT_IF_NEEDED();
6696	HMVMX_ASSERT_CPU_SAFE();
6697	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6698	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6699
6700	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6701	and done this from the VMXR0ThreadCtxCallback(). */
6702	if (!pVCpu->hm.s.fLeaveDone)
6703	{
6704	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6705	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6706	pVCpu->hm.s.fLeaveDone = true;
6707	}
6708
6709	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6710	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6711	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6712	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6713	VMMR0ThreadCtxHooksDeregister(pVCpu);
6714
6715	/* Leave HM context. This takes care of local init (term). */
6716	int rc = HMR0LeaveCpu(pVCpu);
6717
6718	HM_RESTORE_PREEMPT_IF_NEEDED();
6719
6720	return rc;
6721	}
6722
6723
6724	/**
6725	* Does the necessary state syncing before doing a longjmp to ring-3.
6726	*
6727	* @returns VBox status code.
6728	* @param pVM Pointer to the VM.
6729	* @param pVCpu Pointer to the VMCPU.
6730	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6731	* out-of-sync. Make sure to update the required fields
6732	* before using them.
6733	*
6734	* @remarks No-long-jmp zone!!!
6735	*/
6736	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6737	{
6738	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6739	}
6740
6741
6742	/**
6743	* Take necessary actions before going back to ring-3.
6744	*
6745	* An action requires us to go back to ring-3. This function does the necessary
6746	* steps before we can safely return to ring-3. This is not the same as longjmps
6747	* to ring-3, this is voluntary and prepares the guest so it may continue
6748	* executing outside HM (recompiler/IEM).
6749	*
6750	* @returns VBox status code.
6751	* @param pVM Pointer to the VM.
6752	* @param pVCpu Pointer to the VMCPU.
6753	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6754	* out-of-sync. Make sure to update the required fields
6755	* before using them.
6756	* @param rcExit The reason for exiting to ring-3. Can be
6757	* VINF_VMM_UNKNOWN_RING3_CALL.
6758	*/
6759	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6760	{
6761	Assert(pVM);
6762	Assert(pVCpu);
6763	Assert(pMixedCtx);
6764	HMVMX_ASSERT_PREEMPT_SAFE();
6765
6766	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6767	{
6768	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6769	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6770	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6771	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6772	}
6773
6774	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6775	VMMRZCallRing3Disable(pVCpu);
6776	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6777
6778	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6779	if (pVCpu->hm.s.Event.fPending)
6780	{
6781	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6782	Assert(!pVCpu->hm.s.Event.fPending);
6783	}
6784
6785	/* Save guest state and restore host state bits. */
6786	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6787	AssertRCReturn(rc, rc);
6788	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6789
6790	/* Sync recompiler state. */
6791	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6792	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6793	\| CPUM_CHANGED_LDTR
6794	\| CPUM_CHANGED_GDTR
6795	\| CPUM_CHANGED_IDTR
6796	\| CPUM_CHANGED_TR
6797	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6798	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6799	if ( pVM->hm.s.fNestedPaging
6800	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6801	{
6802	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6803	}
6804
6805	Assert(!pVCpu->hm.s.fClearTrapFlag);
6806
6807	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6808	if (rcExit != VINF_EM_RAW_INTERRUPT)
6809	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6810
6811	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6812
6813	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6814	VMMRZCallRing3RemoveNotification(pVCpu);
6815	VMMRZCallRing3Enable(pVCpu);
6816
6817	return rc;
6818	}
6819
6820
6821	/**
6822	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6823	* longjump to ring-3 and possibly get preempted.
6824	*
6825	* @returns VBox status code.
6826	* @param pVCpu Pointer to the VMCPU.
6827	* @param enmOperation The operation causing the ring-3 longjump.
6828	* @param pvUser Opaque pointer to the guest-CPU context. The data
6829	* may be out-of-sync. Make sure to update the required
6830	* fields before using them.
6831	*
6832	* @remarks If you modify code here, make sure to check whether
6833	* hmR0VmxLeave() needs to be updated too!!!
6834	*/
6835	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6836	{
6837	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6838	{
6839	/* If anything here asserts or fails, good luck. */
6840	VMMRZCallRing3RemoveNotification(pVCpu);
6841	VMMRZCallRing3Disable(pVCpu);
6842	HM_DISABLE_PREEMPT_IF_NEEDED();
6843
6844	PVM pVM = pVCpu->CTX_SUFF(pVM);
6845	if (CPUMIsGuestFPUStateActive(pVCpu))
6846	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6847
6848	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6849
6850	#if HC_ARCH_BITS == 64
6851	/* Restore host-state bits that VT-x only restores partially. */
6852	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6853	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6854	{
6855	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6856	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6857	}
6858	#endif
6859
6860	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6861	/* Restore the host MSRs as we're leaving VT-x context. */
6862	if ( HMVMX_IS_64BIT_HOST_MODE()
6863	&& pVM->hm.s.fAllow64BitGuests
6864	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6865	{
6866	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6867	}
6868	#endif
6869	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6870	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6871	{
6872	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6873	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6874	}
6875
6876	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6877	VMMR0ThreadCtxHooksDeregister(pVCpu);
6878
6879	HMR0LeaveCpu(pVCpu);
6880	HM_RESTORE_PREEMPT_IF_NEEDED();
6881	return VINF_SUCCESS;
6882	}
6883
6884	Assert(pVCpu);
6885	Assert(pvUser);
6886	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6887	HMVMX_ASSERT_PREEMPT_SAFE();
6888
6889	VMMRZCallRing3Disable(pVCpu);
6890	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6891
6892	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6893	enmOperation));
6894
6895	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6896	AssertRCReturn(rc, rc);
6897
6898	VMMRZCallRing3Enable(pVCpu);
6899	return VINF_SUCCESS;
6900	}
6901
6902
6903	/**
6904	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6905	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6906	*
6907	* @param pVCpu Pointer to the VMCPU.
6908	*/
6909	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6910	{
6911	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6912	{
6913	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6914	{
6915	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6916	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6917	AssertRC(rc);
6918	}
6919	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6920	}
6921
6922
6923	/**
6924	* Evaluates the event to be delivered to the guest and sets it as the pending
6925	* event.
6926	*
6927	* @param pVCpu Pointer to the VMCPU.
6928	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6929	* out-of-sync. Make sure to update the required fields
6930	* before using them.
6931	*/
6932	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6933	{
6934	Assert(!pVCpu->hm.s.Event.fPending);
6935
6936	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6937	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6938	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6939	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6940
6941	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6942	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6943	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6944	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6945	Assert(!TRPMHasTrap(pVCpu));
6946
6947	/** @todo SMI. SMIs take priority over NMIs. */
6948	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6949	{
6950	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6951	if ( !fBlockMovSS
6952	&& !fBlockSti)
6953	{
6954	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6955	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6956	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6957	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6958
6959	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6960	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6961	}
6962	else
6963	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6964	}
6965	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6966	&& !pVCpu->hm.s.fSingleInstruction)
6967	{
6968	/*
6969	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6970	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6971	* evaluated here and not set as pending, solely based on the force-flags.
6972	*/
6973	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6974	AssertRC(rc);
6975	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6976	if ( !fBlockInt
6977	&& !fBlockSti
6978	&& !fBlockMovSS)
6979	{
6980	uint8_t u8Interrupt;
6981	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6982	if (RT_SUCCESS(rc))
6983	{
6984	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6985	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6986	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6987
6988	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6989	}
6990	else
6991	{
6992	/** @todo Does this actually happen? If not turn it into an assertion. */
6993	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6994	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6995	}
6996	}
6997	else
6998	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6999	}
7000	}
7001
7002
7003	/**
7004	* Injects any pending events into the guest if the guest is in a state to
7005	* receive them.
7006	*
7007	* @returns VBox status code (informational status codes included).
7008	* @param pVCpu Pointer to the VMCPU.
7009	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7010	* out-of-sync. Make sure to update the required fields
7011	* before using them.
7012	*/
7013	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7014	{
7015	HMVMX_ASSERT_PREEMPT_SAFE();
7016	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7017
7018	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7019	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7020	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7021	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7022
7023	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
7024	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7025	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7026	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7027	Assert(!TRPMHasTrap(pVCpu));
7028
7029	int rc = VINF_SUCCESS;
7030	if (pVCpu->hm.s.Event.fPending)
7031	{
7032	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7033	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7034	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7035	{
7036	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7037	AssertRCReturn(rc, rc);
7038	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7039	Assert(!fBlockInt);
7040	Assert(!fBlockSti);
7041	Assert(!fBlockMovSS);
7042	}
7043	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7044	{
7045	Assert(!fBlockSti);
7046	Assert(!fBlockMovSS);
7047	}
7048	#endif
7049	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7050	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7051	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7052	AssertRCReturn(rc, rc);
7053
7054	/* Update the interruptibility-state as it could have been changed by
7055	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7056	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7057	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7058
7059	#ifdef VBOX_WITH_STATISTICS
7060	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7061	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7062	else
7063	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7064	#endif
7065	}
7066
7067	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7068	if ( !pVCpu->hm.s.fSingleInstruction
7069	&& !DBGFIsStepping(pVCpu))
7070	{
7071	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7072	AssertRCReturn(rc2, rc2);
7073	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7074	{
7075	/*
7076	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7077	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7078	* See Intel spec. 27.3.4 "Saving Non-Register State".
7079	*/
7080	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7081	AssertRCReturn(rc2, rc2);
7082	}
7083	}
7084	else
7085	{
7086	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
7087	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
7088	uIntrState = 0;
7089	}
7090
7091	/*
7092	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7093	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7094	*/
7095	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7096	AssertRC(rc2);
7097
7098	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7099	NOREF(fBlockMovSS); NOREF(fBlockSti);
7100	return rc;
7101	}
7102
7103
7104	/**
7105	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7106	*
7107	* @param pVCpu Pointer to the VMCPU.
7108	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7109	* out-of-sync. Make sure to update the required fields
7110	* before using them.
7111	*/
7112	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7113	{
7114	NOREF(pMixedCtx);
7115	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7116	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7117	}
7118
7119
7120	/**
7121	* Injects a double-fault (#DF) exception into the VM.
7122	*
7123	* @returns VBox status code (informational status code included).
7124	* @param pVCpu Pointer to the VMCPU.
7125	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7126	* out-of-sync. Make sure to update the required fields
7127	* before using them.
7128	*/
7129	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7130	{
7131	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7132	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7133	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7134	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7135	puIntrState);
7136	}
7137
7138
7139	/**
7140	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7141	*
7142	* @param pVCpu Pointer to the VMCPU.
7143	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7144	* out-of-sync. Make sure to update the required fields
7145	* before using them.
7146	*/
7147	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7148	{
7149	NOREF(pMixedCtx);
7150	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7151	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7152	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7153	}
7154
7155
7156	/**
7157	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7158	*
7159	* @param pVCpu Pointer to the VMCPU.
7160	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7161	* out-of-sync. Make sure to update the required fields
7162	* before using them.
7163	* @param cbInstr The value of RIP that is to be pushed on the guest
7164	* stack.
7165	*/
7166	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7167	{
7168	NOREF(pMixedCtx);
7169	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7170	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7171	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7172	}
7173
7174
7175	/**
7176	* Injects a general-protection (#GP) fault into the VM.
7177	*
7178	* @returns VBox status code (informational status code included).
7179	* @param pVCpu Pointer to the VMCPU.
7180	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7181	* out-of-sync. Make sure to update the required fields
7182	* before using them.
7183	* @param u32ErrorCode The error code associated with the #GP.
7184	*/
7185	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7186	uint32_t *puIntrState)
7187	{
7188	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7189	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7190	if (fErrorCodeValid)
7191	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7192	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7193	puIntrState);
7194	}
7195
7196
7197	/**
7198	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7199	*
7200	* @param pVCpu Pointer to the VMCPU.
7201	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7202	* out-of-sync. Make sure to update the required fields
7203	* before using them.
7204	* @param uVector The software interrupt vector number.
7205	* @param cbInstr The value of RIP that is to be pushed on the guest
7206	* stack.
7207	*/
7208	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7209	{
7210	NOREF(pMixedCtx);
7211	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7212	if ( uVector == X86_XCPT_BP
7213	\|\| uVector == X86_XCPT_OF)
7214	{
7215	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7216	}
7217	else
7218	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7219	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7220	}
7221
7222
7223	/**
7224	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7225	* stack.
7226	*
7227	* @returns VBox status code (information status code included).
7228	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7229	* @param pVM Pointer to the VM.
7230	* @param pMixedCtx Pointer to the guest-CPU context.
7231	* @param uValue The value to push to the guest stack.
7232	*/
7233	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7234	{
7235	/*
7236	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7237	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7238	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7239	*/
7240	if (pMixedCtx->sp == 1)
7241	return VINF_EM_RESET;
7242	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7243	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7244	AssertRCReturn(rc, rc);
7245	return rc;
7246	}
7247
7248
7249	/**
7250	* Injects an event into the guest upon VM-entry by updating the relevant fields
7251	* in the VM-entry area in the VMCS.
7252	*
7253	* @returns VBox status code (informational error codes included).
7254	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7255	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7256	*
7257	* @param pVCpu Pointer to the VMCPU.
7258	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7259	* be out-of-sync. Make sure to update the required
7260	* fields before using them.
7261	* @param u64IntInfo The VM-entry interruption-information field.
7262	* @param cbInstr The VM-entry instruction length in bytes (for
7263	* software interrupts, exceptions and privileged
7264	* software exceptions).
7265	* @param u32ErrCode The VM-entry exception error code.
7266	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7267	* @param puIntrState Pointer to the current guest interruptibility-state.
7268	* This interruptibility-state will be updated if
7269	* necessary. This cannot not be NULL.
7270	*
7271	* @remarks Requires CR0!
7272	* @remarks No-long-jump zone!!!
7273	*/
7274	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7275	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7276	{
7277	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7278	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7279	Assert(puIntrState);
7280	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7281
7282	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7283	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7284
7285	#ifdef VBOX_STRICT
7286	/* Validate the error-code-valid bit for hardware exceptions. */
7287	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7288	{
7289	switch (uVector)
7290	{
7291	case X86_XCPT_PF:
7292	case X86_XCPT_DF:
7293	case X86_XCPT_TS:
7294	case X86_XCPT_NP:
7295	case X86_XCPT_SS:
7296	case X86_XCPT_GP:
7297	case X86_XCPT_AC:
7298	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7299	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7300	/* fallthru */
7301	default:
7302	break;
7303	}
7304	}
7305	#endif
7306
7307	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7308	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7309	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7310
7311	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7312
7313	/* We require CR0 to check if the guest is in real-mode. */
7314	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7315	AssertRCReturn(rc, rc);
7316
7317	/*
7318	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7319	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7320	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7321	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7322	*/
7323	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7324	{
7325	PVM pVM = pVCpu->CTX_SUFF(pVM);
7326	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7327	{
7328	Assert(PDMVmmDevHeapIsEnabled(pVM));
7329	Assert(pVM->hm.s.vmx.pRealModeTSS);
7330
7331	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7332	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7333	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7334	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7335	AssertRCReturn(rc, rc);
7336	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7337
7338	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7339	const size_t cbIdtEntry = sizeof(X86IDTR16);
7340	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7341	{
7342	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7343	if (uVector == X86_XCPT_DF)
7344	return VINF_EM_RESET;
7345	else if (uVector == X86_XCPT_GP)
7346	{
7347	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7348	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7349	}
7350
7351	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7352	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7353	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7354	}
7355
7356	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7357	uint16_t uGuestIp = pMixedCtx->ip;
7358	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7359	{
7360	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7361	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7362	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7363	}
7364	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7365	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7366
7367	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7368	X86IDTR16 IdtEntry;
7369	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7370	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7371	AssertRCReturn(rc, rc);
7372
7373	/* Construct the stack frame for the interrupt/exception handler. */
7374	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7375	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7376	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7377	AssertRCReturn(rc, rc);
7378
7379	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7380	if (rc == VINF_SUCCESS)
7381	{
7382	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7383	pMixedCtx->rip = IdtEntry.offSel;
7384	pMixedCtx->cs.Sel = IdtEntry.uSel;
7385	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7386	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7387	&& uVector == X86_XCPT_PF)
7388	{
7389	pMixedCtx->cr2 = GCPtrFaultAddress;
7390	}
7391
7392	/* If any other guest-state bits are changed here, make sure to update
7393	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7394	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7395	\| HM_CHANGED_GUEST_RIP
7396	\| HM_CHANGED_GUEST_RFLAGS
7397	\| HM_CHANGED_GUEST_RSP);
7398
7399	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7400	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7401	{
7402	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7403	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7404	Log4(("Clearing inhibition due to STI.\n"));
7405	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7406	}
7407	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7408
7409	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7410	it, if we are returning to ring-3 before executing guest code. */
7411	pVCpu->hm.s.Event.fPending = false;
7412	}
7413	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7414	return rc;
7415	}
7416	else
7417	{
7418	/*
7419	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7420	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7421	*/
7422	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7423	}
7424	}
7425
7426	/* Validate. */
7427	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7428	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7429	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7430
7431	/* Inject. */
7432	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7433	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7434	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7435	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7436
7437	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7438	&& uVector == X86_XCPT_PF)
7439	{
7440	pMixedCtx->cr2 = GCPtrFaultAddress;
7441	}
7442
7443	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7444	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7445
7446	AssertRCReturn(rc, rc);
7447	return rc;
7448	}
7449
7450
7451	/**
7452	* Clears the interrupt-window exiting control in the VMCS and if necessary
7453	* clears the current event in the VMCS as well.
7454	*
7455	* @returns VBox status code.
7456	* @param pVCpu Pointer to the VMCPU.
7457	*
7458	* @remarks Use this function only to clear events that have not yet been
7459	* delivered to the guest but are injected in the VMCS!
7460	* @remarks No-long-jump zone!!!
7461	*/
7462	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7463	{
7464	int rc;
7465	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7466
7467	/* Clear interrupt-window exiting control. */
7468	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7469	{
7470	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7472	AssertRC(rc);
7473	}
7474
7475	if (!pVCpu->hm.s.Event.fPending)
7476	return;
7477
7478	#ifdef VBOX_STRICT
7479	uint32_t u32EntryInfo;
7480	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7481	AssertRC(rc);
7482	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7483	#endif
7484
7485	/* Clear the entry-interruption field (including the valid bit). */
7486	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7487	AssertRC(rc);
7488
7489	/* Clear the pending debug exception field. */
7490	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7491	AssertRC(rc);
7492	}
7493
7494
7495	/**
7496	* Enters the VT-x session.
7497	*
7498	* @returns VBox status code.
7499	* @param pVM Pointer to the VM.
7500	* @param pVCpu Pointer to the VMCPU.
7501	* @param pCpu Pointer to the CPU info struct.
7502	*/
7503	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7504	{
7505	AssertPtr(pVM);
7506	AssertPtr(pVCpu);
7507	Assert(pVM->hm.s.vmx.fSupported);
7508	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7509	NOREF(pCpu); NOREF(pVM);
7510
7511	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7512	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7513
7514	#ifdef VBOX_STRICT
7515	/* Make sure we're in VMX root mode. */
7516	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7517	if (!(u32HostCR4 & X86_CR4_VMXE))
7518	{
7519	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7520	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7521	}
7522	#endif
7523
7524	/*
7525	* Load the VCPU's VMCS as the current (and active) one.
7526	*/
7527	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7528	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7529	if (RT_FAILURE(rc))
7530	return rc;
7531
7532	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7533	pVCpu->hm.s.fLeaveDone = false;
7534	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7535
7536	return VINF_SUCCESS;
7537	}
7538
7539
7540	/**
7541	* The thread-context callback (only on platforms which support it).
7542	*
7543	* @param enmEvent The thread-context event.
7544	* @param pVCpu Pointer to the VMCPU.
7545	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7546	* @thread EMT(pVCpu)
7547	*/
7548	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7549	{
7550	NOREF(fGlobalInit);
7551
7552	switch (enmEvent)
7553	{
7554	case RTTHREADCTXEVENT_PREEMPTING:
7555	{
7556	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7557	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7558	VMCPU_ASSERT_EMT(pVCpu);
7559
7560	PVM pVM = pVCpu->CTX_SUFF(pVM);
7561	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7562
7563	/* No longjmps (logger flushes, locks) in this fragile context. */
7564	VMMRZCallRing3Disable(pVCpu);
7565	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7566
7567	/*
7568	* Restore host-state (FPU, debug etc.)
7569	*/
7570	if (!pVCpu->hm.s.fLeaveDone)
7571	{
7572	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7573	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7574	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7575	pVCpu->hm.s.fLeaveDone = true;
7576	}
7577
7578	/* Leave HM context, takes care of local init (term). */
7579	int rc = HMR0LeaveCpu(pVCpu);
7580	AssertRC(rc); NOREF(rc);
7581
7582	/* Restore longjmp state. */
7583	VMMRZCallRing3Enable(pVCpu);
7584	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7585	break;
7586	}
7587
7588	case RTTHREADCTXEVENT_RESUMED:
7589	{
7590	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7591	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7592	VMCPU_ASSERT_EMT(pVCpu);
7593
7594	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7595	VMMRZCallRing3Disable(pVCpu);
7596	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7597
7598	/* Initialize the bare minimum state required for HM. This takes care of
7599	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7600	int rc = HMR0EnterCpu(pVCpu);
7601	AssertRC(rc);
7602	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7603
7604	/* Load the active VMCS as the current one. */
7605	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7606	{
7607	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7608	AssertRC(rc); NOREF(rc);
7609	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7610	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7611	}
7612	pVCpu->hm.s.fLeaveDone = false;
7613
7614	/* Restore longjmp state. */
7615	VMMRZCallRing3Enable(pVCpu);
7616	break;
7617	}
7618
7619	default:
7620	break;
7621	}
7622	}
7623
7624
7625	/**
7626	* Saves the host state in the VMCS host-state.
7627	* Sets up the VM-exit MSR-load area.
7628	*
7629	* The CPU state will be loaded from these fields on every successful VM-exit.
7630	*
7631	* @returns VBox status code.
7632	* @param pVM Pointer to the VM.
7633	* @param pVCpu Pointer to the VMCPU.
7634	*
7635	* @remarks No-long-jump zone!!!
7636	*/
7637	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7638	{
7639	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7640
7641	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7642	return VINF_SUCCESS;
7643
7644	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7645	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7646
7647	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7648	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7649
7650	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7651	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7652
7653	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7654	return rc;
7655	}
7656
7657
7658	/**
7659	* Saves the host state in the VMCS host-state.
7660	*
7661	* @returns VBox status code.
7662	* @param pVM Pointer to the VM.
7663	* @param pVCpu Pointer to the VMCPU.
7664	*
7665	* @remarks No-long-jump zone!!!
7666	*/
7667	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7668	{
7669	AssertPtr(pVM);
7670	AssertPtr(pVCpu);
7671
7672	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7673
7674	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7675	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7676	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7677	return hmR0VmxSaveHostState(pVM, pVCpu);
7678	}
7679
7680
7681	/**
7682	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7683	* loaded from these fields on every successful VM-entry.
7684	*
7685	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7686	* Sets up the VM-entry controls.
7687	* Sets up the appropriate VMX non-root function to execute guest code based on
7688	* the guest CPU mode.
7689	*
7690	* @returns VBox status code.
7691	* @param pVM Pointer to the VM.
7692	* @param pVCpu Pointer to the VMCPU.
7693	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7694	* out-of-sync. Make sure to update the required fields
7695	* before using them.
7696	*
7697	* @remarks No-long-jump zone!!!
7698	*/
7699	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7700	{
7701	AssertPtr(pVM);
7702	AssertPtr(pVCpu);
7703	AssertPtr(pMixedCtx);
7704	HMVMX_ASSERT_PREEMPT_SAFE();
7705
7706	#ifdef LOG_ENABLED
7707	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7708	* probably not initialized yet? Anyway this will do for now.
7709	*
7710	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7711	* interface and disable ring-3 calls when thread-context hooks are not
7712	* available. */
7713	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7714	VMMR0LogFlushDisable(pVCpu);
7715	#endif
7716
7717	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7718
7719	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7720
7721	/* Determine real-on-v86 mode. */
7722	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7723	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7724	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7725	{
7726	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7727	}
7728
7729	/*
7730	* Load the guest-state into the VMCS.
7731	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7732	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7733	*/
7734	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7735	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7736
7737	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7738	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7739	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7740
7741	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7742	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7743	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7744
7745	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7746	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7747
7748	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7749	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7750
7751	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7752	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7753	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7754
7755	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7756	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7757
7758	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7759	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7760
7761	/*
7762	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7763	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7764	*/
7765	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7766	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7767
7768	/* Clear any unused and reserved bits. */
7769	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7770
7771	#ifdef LOG_ENABLED
7772	/* Only reenable log-flushing if the caller has it enabled. */
7773	if (!fCallerDisabledLogFlush)
7774	VMMR0LogFlushEnable(pVCpu);
7775	#endif
7776
7777	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7778	return rc;
7779	}
7780
7781
7782	/**
7783	* Loads the state shared between the host and guest into the VMCS.
7784	*
7785	* @param pVM Pointer to the VM.
7786	* @param pVCpu Pointer to the VMCPU.
7787	* @param pCtx Pointer to the guest-CPU context.
7788	*
7789	* @remarks No-long-jump zone!!!
7790	*/
7791	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7792	{
7793	NOREF(pVM);
7794
7795	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7796	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7797
7798	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7799	{
7800	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7801	AssertRC(rc);
7802	}
7803
7804	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7805	{
7806	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7807	AssertRC(rc);
7808
7809	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7810	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7811	{
7812	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7813	AssertRC(rc);
7814	}
7815	}
7816
7817	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7818	{
7819	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7820	if ( HMVMX_IS_64BIT_HOST_MODE()
7821	&& pVM->hm.s.fAllow64BitGuests)
7822	{
7823	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7824	}
7825	#endif
7826	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7827	}
7828
7829	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7830	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7831	}
7832
7833
7834	/**
7835	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7836	*
7837	* @param pVM Pointer to the VM.
7838	* @param pVCpu Pointer to the VMCPU.
7839	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7840	* out-of-sync. Make sure to update the required fields
7841	* before using them.
7842	*/
7843	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7844	{
7845	HMVMX_ASSERT_PREEMPT_SAFE();
7846
7847	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7848	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7849	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7850	#endif
7851
7852	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7853	{
7854	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7855	AssertRC(rc);
7856	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7857	}
7858	else if (VMCPU_HMCF_VALUE(pVCpu))
7859	{
7860	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7861	AssertRC(rc);
7862	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7863	}
7864
7865	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7866	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7867	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7868	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7869
7870	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7871	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7872	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7873	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7874	#endif
7875	}
7876
7877
7878	/**
7879	* Does the preparations before executing guest code in VT-x.
7880	*
7881	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7882	* recompiler. We must be cautious what we do here regarding committing
7883	* guest-state information into the VMCS assuming we assuredly execute the
7884	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7885	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7886	* so that the recompiler can (and should) use them when it resumes guest
7887	* execution. Otherwise such operations must be done when we can no longer
7888	* exit to ring-3.
7889	*
7890	* @returns Strict VBox status code.
7891	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7892	* have been disabled.
7893	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7894	* double-fault into the guest.
7895	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7896	*
7897	* @param pVM Pointer to the VM.
7898	* @param pVCpu Pointer to the VMCPU.
7899	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7900	* out-of-sync. Make sure to update the required fields
7901	* before using them.
7902	* @param pVmxTransient Pointer to the VMX transient structure.
7903	*
7904	* @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7905	* interrupts will be disabled.
7906	*/
7907	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7908	{
7909	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7910
7911	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7912	PGMRZDynMapFlushAutoSet(pVCpu);
7913	#endif
7914
7915	/* Check force flag actions that might require us to go back to ring-3. */
7916	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7917	if (rc != VINF_SUCCESS)
7918	return rc;
7919
7920	#ifndef IEM_VERIFICATION_MODE_FULL
7921	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7922	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7923	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7924	{
7925	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7926	RTGCPHYS GCPhysApicBase;
7927	GCPhysApicBase = pMixedCtx->msrApicBase;
7928	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7929
7930	/* Unalias any existing mapping. */
7931	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7932	AssertRCReturn(rc, rc);
7933
7934	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7935	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7936	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7937	AssertRCReturn(rc, rc);
7938
7939	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7940	}
7941	#endif /* !IEM_VERIFICATION_MODE_FULL */
7942
7943	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7944	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7945
7946	/*
7947	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7948	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7949	*/
7950	if (TRPMHasTrap(pVCpu))
7951	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7952	else if (!pVCpu->hm.s.Event.fPending)
7953	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7954
7955	/*
7956	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7957	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7958	*/
7959	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7960	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7961	{
7962	Assert(rc == VINF_EM_RESET);
7963	return rc;
7964	}
7965
7966	/*
7967	* No longjmps to ring-3 from this point on!!!
7968	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7969	* This also disables flushing of the R0-logger instance (if any).
7970	*/
7971	VMMRZCallRing3Disable(pVCpu);
7972
7973	/*
7974	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7975	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7976	*
7977	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7978	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7979	*
7980	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7981	* executing guest code.
7982	*/
7983	pVmxTransient->uEflags = ASMIntDisableFlags();
7984	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7985	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7986	{
7987	hmR0VmxClearEventVmcs(pVCpu);
7988	ASMSetFlags(pVmxTransient->uEflags);
7989	VMMRZCallRing3Enable(pVCpu);
7990	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7991	return VINF_EM_RAW_TO_R3;
7992	}
7993	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7994	{
7995	hmR0VmxClearEventVmcs(pVCpu);
7996	ASMSetFlags(pVmxTransient->uEflags);
7997	VMMRZCallRing3Enable(pVCpu);
7998	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7999	return VINF_EM_RAW_INTERRUPT;
8000	}
8001
8002	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8003	pVCpu->hm.s.Event.fPending = false;
8004
8005	return VINF_SUCCESS;
8006	}
8007
8008
8009	/**
8010	* Prepares to run guest code in VT-x and we've committed to doing so. This
8011	* means there is no backing out to ring-3 or anywhere else at this
8012	* point.
8013	*
8014	* @param pVM Pointer to the VM.
8015	* @param pVCpu Pointer to the VMCPU.
8016	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8017	* out-of-sync. Make sure to update the required fields
8018	* before using them.
8019	* @param pVmxTransient Pointer to the VMX transient structure.
8020	*
8021	* @remarks Called with preemption disabled.
8022	* @remarks No-long-jump zone!!!
8023	*/
8024	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8025	{
8026	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8027	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8028	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8029
8030	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8031	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8032
8033	/*
8034	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8035	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8036	* Reload only the necessary state, the assertion will catch if other parts of the code
8037	* change.
8038	*/
8039	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8040	{
8041	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8042	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8043	}
8044
8045	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8046	if (!CPUMIsGuestFPUStateActive(pVCpu))
8047	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8048	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8049	#endif
8050
8051	if ( pVCpu->hm.s.fUseGuestFpu
8052	&& !CPUMIsGuestFPUStateActive(pVCpu))
8053	{
8054	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8055	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
8056	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8057	}
8058
8059	/*
8060	* The host MSR values the very first time around won't be updated, so we need to
8061	* fill those values in. Subsequently, it's updated as part of the host state.
8062	*/
8063	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8064	&& pVCpu->hm.s.vmx.cMsrs > 0)
8065	{
8066	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8067	}
8068
8069	/*
8070	* Load the host state bits as we may've been preempted (only happens when
8071	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8072	*/
8073	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8074	{
8075	/* This ASSUMES that pfnStartVM has been set up already. */
8076	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8077	AssertRC(rc);
8078	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8079	}
8080	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8081
8082	/*
8083	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8084	*/
8085	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8086	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8087	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
8088
8089	/* Store status of the shared guest-host state at the time of VM-entry. */
8090	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8091	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8092	{
8093	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8094	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8095	}
8096	else
8097	#endif
8098	{
8099	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8100	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8101	}
8102	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8103
8104	/*
8105	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8106	*/
8107	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8108	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8109
8110	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8111	RTCPUID idCurrentCpu = pCpu->idCpu;
8112	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8113	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8114	{
8115	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8116	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8117	}
8118
8119	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8120	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8121	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8122	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8123
8124	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8125
8126	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8127	to start executing. */
8128
8129	/*
8130	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8131	*/
8132	uint64_t uGuestTscAuxMsr;
8133	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8134	{
8135	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8136	{
8137	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &uGuestTscAuxMsr);
8138	AssertRC(rc2);
8139	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, uGuestTscAuxMsr, true /* fUpdateHostMsr */);
8140	}
8141	else
8142	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8143	}
8144	#ifdef VBOX_STRICT
8145	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8146	#endif
8147	}
8148
8149
8150	/**
8151	* Performs some essential restoration of state after running guest code in
8152	* VT-x.
8153	*
8154	* @param pVM Pointer to the VM.
8155	* @param pVCpu Pointer to the VMCPU.
8156	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8157	* out-of-sync. Make sure to update the required fields
8158	* before using them.
8159	* @param pVmxTransient Pointer to the VMX transient structure.
8160	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8161	*
8162	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8163	*
8164	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8165	* unconditionally when it is safe to do so.
8166	*/
8167	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8168	{
8169	NOREF(pVM);
8170
8171	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8172
8173	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8174	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8175	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8176	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8177	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8178
8179	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8180	{
8181	/** @todo Find a way to fix hardcoding a guestimate. */
8182	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8183	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8184	}
8185
8186	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8187	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8188	Assert(!(ASMGetFlags() & X86_EFL_IF));
8189	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8190
8191	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8192	if (CPUMIsGuestFPUStateActive(pVCpu))
8193	{
8194	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8195	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8196	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8197	}
8198	#endif
8199
8200	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8201	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8202	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8203	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8204
8205	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8206	uint32_t uExitReason;
8207	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8208	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8209	AssertRC(rc);
8210	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8211	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8212
8213	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8214	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8215	{
8216	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8217	pVmxTransient->fVMEntryFailed));
8218	return;
8219	}
8220
8221	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8222	{
8223	/* Update the guest interruptibility-state from the VMCS. */
8224	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8225	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8226	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8227	AssertRC(rc);
8228	#endif
8229	/*
8230	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8231	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8232	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8233	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8234	*/
8235	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8236	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8237	{
8238	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8239	AssertRC(rc);
8240	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8241	}
8242	}
8243	}
8244
8245
8246
8247	/**
8248	* Runs the guest code using VT-x the normal way.
8249	*
8250	* @returns VBox status code.
8251	* @param pVM Pointer to the VM.
8252	* @param pVCpu Pointer to the VMCPU.
8253	* @param pCtx Pointer to the guest-CPU context.
8254	*
8255	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
8256	* @remarks Called with preemption disabled.
8257	*/
8258	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8259	{
8260	VMXTRANSIENT VmxTransient;
8261	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8262	int rc = VERR_INTERNAL_ERROR_5;
8263	uint32_t cLoops = 0;
8264
8265	for (;; cLoops++)
8266	{
8267	Assert(!HMR0SuspendPending());
8268	HMVMX_ASSERT_CPU_SAFE();
8269
8270	/* Preparatory work for running guest code, this may force us to return
8271	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8272	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8273	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8274	if (rc != VINF_SUCCESS)
8275	break;
8276
8277	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8278	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8279	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8280
8281	/* Restore any residual host-state and save any bits shared between host
8282	and guest into the guest-CPU state. Re-enables interrupts! */
8283	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8284
8285	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8286	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8287	{
8288	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8289	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8290	return rc;
8291	}
8292
8293	/* Handle the VM-exit. */
8294	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8295	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8296	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8297	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8298	HMVMX_START_EXIT_DISPATCH_PROF();
8299	#ifdef HMVMX_USE_FUNCTION_TABLE
8300	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8301	#else
8302	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8303	#endif
8304	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8305	if (rc != VINF_SUCCESS)
8306	break;
8307	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8308	{
8309	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8310	rc = VINF_EM_RAW_INTERRUPT;
8311	break;
8312	}
8313	}
8314
8315	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8316	return rc;
8317	}
8318
8319
8320	/**
8321	* Single steps guest code using VT-x.
8322	*
8323	* @returns VBox status code.
8324	* @param pVM Pointer to the VM.
8325	* @param pVCpu Pointer to the VMCPU.
8326	* @param pCtx Pointer to the guest-CPU context.
8327	*
8328	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
8329	* @remarks Called with preemption disabled.
8330	*/
8331	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8332	{
8333	VMXTRANSIENT VmxTransient;
8334	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8335	int rc = VERR_INTERNAL_ERROR_5;
8336	uint32_t cLoops = 0;
8337	uint16_t uCsStart = pCtx->cs.Sel;
8338	uint64_t uRipStart = pCtx->rip;
8339
8340	for (;; cLoops++)
8341	{
8342	Assert(!HMR0SuspendPending());
8343	HMVMX_ASSERT_CPU_SAFE();
8344
8345	/* Preparatory work for running guest code, this may force us to return
8346	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8347	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8348	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8349	if (rc != VINF_SUCCESS)
8350	break;
8351
8352	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8353	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8354	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8355
8356	/* Restore any residual host-state and save any bits shared between host
8357	and guest into the guest-CPU state. Re-enables interrupts! */
8358	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8359
8360	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8361	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8362	{
8363	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8364	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8365	return rc;
8366	}
8367
8368	/* Handle the VM-exit. */
8369	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8370	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8371	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8372	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8373	HMVMX_START_EXIT_DISPATCH_PROF();
8374	#ifdef HMVMX_USE_FUNCTION_TABLE
8375	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8376	#else
8377	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8378	#endif
8379	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8380	if (rc != VINF_SUCCESS)
8381	break;
8382	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8383	{
8384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8385	rc = VINF_EM_RAW_INTERRUPT;
8386	break;
8387	}
8388
8389	/*
8390	* Did the RIP change, if so, consider it a single step.
8391	* Otherwise, make sure one of the TFs gets set.
8392	*/
8393	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8394	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8395	AssertRCReturn(rc2, rc2);
8396	if ( pCtx->rip != uRipStart
8397	\|\| pCtx->cs.Sel != uCsStart)
8398	{
8399	rc = VINF_EM_DBG_STEPPED;
8400	break;
8401	}
8402	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8403	}
8404
8405	/*
8406	* Clear the X86_EFL_TF if necessary.
8407	*/
8408	if (pVCpu->hm.s.fClearTrapFlag)
8409	{
8410	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8411	AssertRCReturn(rc2, rc2);
8412	pVCpu->hm.s.fClearTrapFlag = false;
8413	pCtx->eflags.Bits.u1TF = 0;
8414	}
8415	/** @todo there seems to be issues with the resume flag when the monitor trap
8416	* flag is pending without being used. Seen early in bios init when
8417	* accessing APIC page in prot mode. */
8418
8419	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8420	return rc;
8421	}
8422
8423
8424	/**
8425	* Runs the guest code using VT-x.
8426	*
8427	* @returns VBox status code.
8428	* @param pVM Pointer to the VM.
8429	* @param pVCpu Pointer to the VMCPU.
8430	* @param pCtx Pointer to the guest-CPU context.
8431	*
8432	* @remarks Called with preemption disabled.
8433	*/
8434	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8435	{
8436	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8437	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8438	HMVMX_ASSERT_PREEMPT_SAFE();
8439
8440	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8441
8442	int rc;
8443	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8444	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8445	else
8446	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8447
8448	if (rc == VERR_EM_INTERPRETER)
8449	rc = VINF_EM_RAW_EMULATE_INSTR;
8450	else if (rc == VINF_EM_RESET)
8451	rc = VINF_EM_TRIPLE_FAULT;
8452
8453	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8454	if (RT_FAILURE(rc2))
8455	{
8456	pVCpu->hm.s.u32HMError = rc;
8457	rc = rc2;
8458	}
8459	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8460	return rc;
8461	}
8462
8463
8464	#ifndef HMVMX_USE_FUNCTION_TABLE
8465	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8466	{
8467	#ifdef DEBUG_ramshankar
8468	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8469	# define LDVMCS() do { VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8470	#endif
8471	int rc;
8472	switch (rcReason)
8473	{
8474	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8475	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8476	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8477	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8478	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8479	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8480	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8481	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8482	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8483	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8484	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8485	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8486	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8487	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8488	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8489	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8490	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8491	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8492	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8493	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8494	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8495	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8496	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8497	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8498	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8499	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8500	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507
8508	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8509	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8510	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8511	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8512	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8513	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8514	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8515	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8516	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8517
8518	case VMX_EXIT_VMCALL:
8519	case VMX_EXIT_VMCLEAR:
8520	case VMX_EXIT_VMLAUNCH:
8521	case VMX_EXIT_VMPTRLD:
8522	case VMX_EXIT_VMPTRST:
8523	case VMX_EXIT_VMREAD:
8524	case VMX_EXIT_VMRESUME:
8525	case VMX_EXIT_VMWRITE:
8526	case VMX_EXIT_VMXOFF:
8527	case VMX_EXIT_VMXON:
8528	case VMX_EXIT_INVEPT:
8529	case VMX_EXIT_INVVPID:
8530	case VMX_EXIT_VMFUNC:
8531	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8532	break;
8533	default:
8534	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8535	break;
8536	}
8537	return rc;
8538	}
8539	#endif
8540
8541	#ifdef DEBUG
8542	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8543	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8544	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8545
8546	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8547	do \
8548	{ \
8549	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8550	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8551	} while (0)
8552
8553	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8554	do { \
8555	AssertPtr(pVCpu); \
8556	AssertPtr(pMixedCtx); \
8557	AssertPtr(pVmxTransient); \
8558	Assert(pVmxTransient->fVMEntryFailed == false); \
8559	Assert(ASMIntAreEnabled()); \
8560	HMVMX_ASSERT_PREEMPT_SAFE(); \
8561	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8562	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8563	HMVMX_ASSERT_PREEMPT_SAFE(); \
8564	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8565	HMVMX_ASSERT_PREEMPT_CPUID(); \
8566	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8567	} while (0)
8568
8569	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8570	do { \
8571	Log4Func(("\n")); \
8572	} while (0)
8573	#else /* Release builds */
8574	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8575	do { \
8576	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8577	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8578	} while (0)
8579	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8580	#endif
8581
8582
8583	/**
8584	* Advances the guest RIP after reading it from the VMCS.
8585	*
8586	* @returns VBox status code.
8587	* @param pVCpu Pointer to the VMCPU.
8588	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8589	* out-of-sync. Make sure to update the required fields
8590	* before using them.
8591	* @param pVmxTransient Pointer to the VMX transient structure.
8592	*
8593	* @remarks No-long-jump zone!!!
8594	*/
8595	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8596	{
8597	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8598	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8599	AssertRCReturn(rc, rc);
8600
8601	pMixedCtx->rip += pVmxTransient->cbInstr;
8602	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8603	return rc;
8604	}
8605
8606
8607	/**
8608	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8609	* and update error record fields accordingly.
8610	*
8611	* @return VMX_IGS_* return codes.
8612	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8613	* wrong with the guest state.
8614	*
8615	* @param pVM Pointer to the VM.
8616	* @param pVCpu Pointer to the VMCPU.
8617	* @param pCtx Pointer to the guest-CPU state.
8618	*/
8619	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8620	{
8621	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8622	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8623	uError = (err); \
8624	break; \
8625	} else do { } while (0)
8626	/* Duplicate of IEM_IS_CANONICAL(). */
8627	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8628
8629	int rc;
8630	uint32_t uError = VMX_IGS_ERROR;
8631	uint32_t u32Val;
8632	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8633
8634	do
8635	{
8636	/*
8637	* CR0.
8638	*/
8639	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8640	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8641	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8642	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8643	if (fUnrestrictedGuest)
8644	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8645
8646	uint32_t u32GuestCR0;
8647	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8648	AssertRCBreak(rc);
8649	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8650	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8651	if ( !fUnrestrictedGuest
8652	&& (u32GuestCR0 & X86_CR0_PG)
8653	&& !(u32GuestCR0 & X86_CR0_PE))
8654	{
8655	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8656	}
8657
8658	/*
8659	* CR4.
8660	*/
8661	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8662	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8663
8664	uint32_t u32GuestCR4;
8665	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8666	AssertRCBreak(rc);
8667	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8668	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8669
8670	/*
8671	* IA32_DEBUGCTL MSR.
8672	*/
8673	uint64_t u64Val;
8674	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8675	AssertRCBreak(rc);
8676	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8677	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8678	{
8679	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8680	}
8681	uint64_t u64DebugCtlMsr = u64Val;
8682
8683	#ifdef VBOX_STRICT
8684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8685	AssertRCBreak(rc);
8686	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8687	#endif
8688	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8689
8690	/*
8691	* RIP and RFLAGS.
8692	*/
8693	uint32_t u32Eflags;
8694	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8695	if (HMVMX_IS_64BIT_HOST_MODE())
8696	{
8697	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8698	AssertRCBreak(rc);
8699	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8700	if ( !fLongModeGuest
8701	\|\| !pCtx->cs.Attr.n.u1Long)
8702	{
8703	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8704	}
8705	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8706	* must be identical if the "IA32e mode guest" VM-entry control is 1
8707	* and CS.L is 1. No check applies if the CPU supports 64
8708	* linear-address bits. */
8709
8710	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8711	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8712	AssertRCBreak(rc);
8713	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8714	VMX_IGS_RFLAGS_RESERVED);
8715	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8716	u32Eflags = u64Val;
8717	}
8718	else
8719	#endif
8720	{
8721	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8722	AssertRCBreak(rc);
8723	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8724	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8725	}
8726
8727	if ( fLongModeGuest
8728	\|\| ( fUnrestrictedGuest
8729	&& !(u32GuestCR0 & X86_CR0_PE)))
8730	{
8731	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8732	}
8733
8734	uint32_t u32EntryInfo;
8735	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8736	AssertRCBreak(rc);
8737	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8738	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8739	{
8740	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8741	}
8742
8743	/*
8744	* 64-bit checks.
8745	*/
8746	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8747	if (HMVMX_IS_64BIT_HOST_MODE())
8748	{
8749	if ( fLongModeGuest
8750	&& !fUnrestrictedGuest)
8751	{
8752	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8753	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8754	}
8755
8756	if ( !fLongModeGuest
8757	&& (u32GuestCR4 & X86_CR4_PCIDE))
8758	{
8759	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8760	}
8761
8762	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8763	* 51:32 beyond the processor's physical-address width are 0. */
8764
8765	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8766	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8767	{
8768	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8769	}
8770
8771	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8772	AssertRCBreak(rc);
8773	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8774
8775	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8776	AssertRCBreak(rc);
8777	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8778	}
8779	#endif
8780
8781	/*
8782	* PERF_GLOBAL MSR.
8783	*/
8784	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8785	{
8786	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8787	AssertRCBreak(rc);
8788	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8789	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8790	}
8791
8792	/*
8793	* PAT MSR.
8794	*/
8795	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8796	{
8797	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8798	AssertRCBreak(rc);
8799	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8800	for (unsigned i = 0; i < 8; i++)
8801	{
8802	uint8_t u8Val = (u64Val & 0x7);
8803	if ( u8Val != 0 /* UC */
8804	\|\| u8Val != 1 /* WC */
8805	\|\| u8Val != 4 /* WT */
8806	\|\| u8Val != 5 /* WP */
8807	\|\| u8Val != 6 /* WB */
8808	\|\| u8Val != 7 /* UC- */)
8809	{
8810	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8811	}
8812	u64Val >>= 3;
8813	}
8814	}
8815
8816	/*
8817	* EFER MSR.
8818	*/
8819	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8820	{
8821	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8822	AssertRCBreak(rc);
8823	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8824	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8825	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8826	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8827	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8828	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8829	}
8830
8831	/*
8832	* Segment registers.
8833	*/
8834	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8835	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8836	if (!(u32Eflags & X86_EFL_VM))
8837	{
8838	/* CS */
8839	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8840	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8841	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8842	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8843	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8844	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8845	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8846	/* CS cannot be loaded with NULL in protected mode. */
8847	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8848	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8849	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8850	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8851	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8852	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8853	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8854	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8855	else
8856	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8857
8858	/* SS */
8859	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8860	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8861	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8862	if ( !(pCtx->cr0 & X86_CR0_PE)
8863	\|\| pCtx->cs.Attr.n.u4Type == 3)
8864	{
8865	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8866	}
8867	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8868	{
8869	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8870	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8871	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8872	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8873	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8874	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8875	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8876	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8877	}
8878
8879	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8880	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8881	{
8882	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8883	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8884	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8885	\|\| pCtx->ds.Attr.n.u4Type > 11
8886	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8887	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8888	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8889	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8890	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8891	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8892	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8893	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8894	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8895	}
8896	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8897	{
8898	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8899	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8900	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8901	\|\| pCtx->es.Attr.n.u4Type > 11
8902	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8903	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8904	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8905	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8906	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8907	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8908	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8909	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8910	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8911	}
8912	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8913	{
8914	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8915	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8916	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8917	\|\| pCtx->fs.Attr.n.u4Type > 11
8918	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8919	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8920	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8921	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8922	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8923	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8924	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8925	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8926	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8927	}
8928	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8929	{
8930	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8931	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8932	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8933	\|\| pCtx->gs.Attr.n.u4Type > 11
8934	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8935	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8936	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8937	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8938	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8939	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8940	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8941	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8942	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8943	}
8944	/* 64-bit capable CPUs. */
8945	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8946	if (HMVMX_IS_64BIT_HOST_MODE())
8947	{
8948	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8949	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8950	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8951	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8952	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8953	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8954	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8955	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8956	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8957	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8958	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8959	}
8960	#endif
8961	}
8962	else
8963	{
8964	/* V86 mode checks. */
8965	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8966	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8967	{
8968	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8969	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8970	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8971	}
8972	else
8973	{
8974	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8975	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8976	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8977	}
8978
8979	/* CS */
8980	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8981	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8982	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8983	/* SS */
8984	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8985	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8986	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8987	/* DS */
8988	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8989	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8990	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8991	/* ES */
8992	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8993	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8994	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8995	/* FS */
8996	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8997	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8998	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8999	/* GS */
9000	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9001	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9002	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9003	/* 64-bit capable CPUs. */
9004	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9005	if (HMVMX_IS_64BIT_HOST_MODE())
9006	{
9007	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9008	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9009	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9010	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9011	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9012	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9013	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9014	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9015	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9016	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9017	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9018	}
9019	#endif
9020	}
9021
9022	/*
9023	* TR.
9024	*/
9025	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9026	/* 64-bit capable CPUs. */
9027	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9028	if (HMVMX_IS_64BIT_HOST_MODE())
9029	{
9030	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9031	}
9032	#endif
9033	if (fLongModeGuest)
9034	{
9035	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9036	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9037	}
9038	else
9039	{
9040	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9041	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9042	VMX_IGS_TR_ATTR_TYPE_INVALID);
9043	}
9044	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9045	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9046	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9047	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9048	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9049	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9050	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9051	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9052
9053	/*
9054	* GDTR and IDTR.
9055	*/
9056	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9057	if (HMVMX_IS_64BIT_HOST_MODE())
9058	{
9059	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9060	AssertRCBreak(rc);
9061	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9062
9063	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9064	AssertRCBreak(rc);
9065	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9066	}
9067	#endif
9068
9069	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9070	AssertRCBreak(rc);
9071	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9072
9073	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9074	AssertRCBreak(rc);
9075	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9076
9077	/*
9078	* Guest Non-Register State.
9079	*/
9080	/* Activity State. */
9081	uint32_t u32ActivityState;
9082	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9083	AssertRCBreak(rc);
9084	HMVMX_CHECK_BREAK( !u32ActivityState
9085	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9086	VMX_IGS_ACTIVITY_STATE_INVALID);
9087	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9088	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9089	uint32_t u32IntrState;
9090	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9091	AssertRCBreak(rc);
9092	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9093	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9094	{
9095	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9096	}
9097
9098	/** @todo Activity state and injecting interrupts. Left as a todo since we
9099	* currently don't use activity states but ACTIVE. */
9100
9101	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9102	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9103
9104	/* Guest interruptibility-state. */
9105	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9106	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9107	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9108	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9109	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9110	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9111	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9112	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9113	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9114	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9115	{
9116	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9117	{
9118	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9119	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9120	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9121	}
9122	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9123	{
9124	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9125	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9126	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9127	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9128	}
9129	}
9130	/** @todo Assumes the processor is not in SMM. */
9131	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9132	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9133	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9134	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9135	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9136	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9137	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9138	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9139	{
9140	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9141	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9142	}
9143
9144	/* Pending debug exceptions. */
9145	if (HMVMX_IS_64BIT_HOST_MODE())
9146	{
9147	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9148	AssertRCBreak(rc);
9149	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9150	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9151	u32Val = u64Val; /* For pending debug exceptions checks below. */
9152	}
9153	else
9154	{
9155	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9156	AssertRCBreak(rc);
9157	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9158	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9159	}
9160
9161	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9162	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9163	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9164	{
9165	if ( (u32Eflags & X86_EFL_TF)
9166	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9167	{
9168	/* Bit 14 is PendingDebug.BS. */
9169	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9170	}
9171	if ( !(u32Eflags & X86_EFL_TF)
9172	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9173	{
9174	/* Bit 14 is PendingDebug.BS. */
9175	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9176	}
9177	}
9178
9179	/* VMCS link pointer. */
9180	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9181	AssertRCBreak(rc);
9182	if (u64Val != UINT64_C(0xffffffffffffffff))
9183	{
9184	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9185	/** @todo Bits beyond the processor's physical-address width MBZ. */
9186	/** @todo 32-bit located in memory referenced by value of this field (as a
9187	* physical address) must contain the processor's VMCS revision ID. */
9188	/** @todo SMM checks. */
9189	}
9190
9191	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
9192
9193	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9194	if (uError == VMX_IGS_ERROR)
9195	uError = VMX_IGS_REASON_NOT_FOUND;
9196	} while (0);
9197
9198	pVCpu->hm.s.u32HMError = uError;
9199	return uError;
9200
9201	#undef HMVMX_ERROR_BREAK
9202	#undef HMVMX_CHECK_BREAK
9203	#undef HMVMX_IS_CANONICAL
9204	}
9205
9206	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9207	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9208	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9209
9210	/** @name VM-exit handlers.
9211	* @{
9212	*/
9213
9214	/**
9215	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9216	*/
9217	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9218	{
9219	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9220	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9221	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9222	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9223	return VINF_SUCCESS;
9224	return VINF_EM_RAW_INTERRUPT;
9225	}
9226
9227
9228	/**
9229	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9230	*/
9231	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9232	{
9233	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9234	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9235
9236	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9237	AssertRCReturn(rc, rc);
9238
9239	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9240	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9241	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9242	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9243
9244	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9245	{
9246	/*
9247	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9248	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9249	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9250	*
9251	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9252	*/
9253	VMXDispatchHostNmi();
9254	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9255	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9256	return VINF_SUCCESS;
9257	}
9258
9259	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9260	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9261	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9262	{
9263	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9264	return VINF_SUCCESS;
9265	}
9266	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9267	{
9268	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9269	return rc;
9270	}
9271
9272	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9273	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9274	switch (uIntType)
9275	{
9276	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9277	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9278	/* no break */
9279	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9280	{
9281	switch (uVector)
9282	{
9283	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9284	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9285	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9286	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9287	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9288	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9289	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9290	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9291	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9292	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9293	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9294	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9295	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9296	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9297	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9298	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9299	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9300	#endif
9301	default:
9302	{
9303	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9304	AssertRCReturn(rc, rc);
9305
9306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9307	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9308	{
9309	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9310	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9311	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9312
9313	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9314	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9315	AssertRCReturn(rc, rc);
9316	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9317	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9318	0 /* GCPtrFaultAddress */);
9319	AssertRCReturn(rc, rc);
9320	}
9321	else
9322	{
9323	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9324	pVCpu->hm.s.u32HMError = uVector;
9325	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9326	}
9327	break;
9328	}
9329	}
9330	break;
9331	}
9332
9333	default:
9334	{
9335	pVCpu->hm.s.u32HMError = uExitIntInfo;
9336	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9337	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9338	break;
9339	}
9340	}
9341	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9342	return rc;
9343	}
9344
9345
9346	/**
9347	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9348	*/
9349	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9350	{
9351	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9352
9353	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9354	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9355	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9356	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9357	AssertRCReturn(rc, rc);
9358
9359	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9360	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9361	return VINF_SUCCESS;
9362	}
9363
9364
9365	/**
9366	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9367	*/
9368	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9369	{
9370	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9371	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9372	HMVMX_RETURN_UNEXPECTED_EXIT();
9373	}
9374
9375
9376	/**
9377	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9378	*/
9379	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9380	{
9381	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9382	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9383	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9384	}
9385
9386
9387	/**
9388	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9389	*/
9390	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9391	{
9392	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9393	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9394	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9395	}
9396
9397
9398	/**
9399	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9400	*/
9401	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9402	{
9403	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9404	PVM pVM = pVCpu->CTX_SUFF(pVM);
9405	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9406	if (RT_LIKELY(rc == VINF_SUCCESS))
9407	{
9408	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9409	Assert(pVmxTransient->cbInstr == 2);
9410	}
9411	else
9412	{
9413	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9414	rc = VERR_EM_INTERPRETER;
9415	}
9416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9417	return rc;
9418	}
9419
9420
9421	/**
9422	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9423	*/
9424	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9425	{
9426	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9427	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9428	AssertRCReturn(rc, rc);
9429
9430	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9431	return VINF_EM_RAW_EMULATE_INSTR;
9432
9433	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9434	HMVMX_RETURN_UNEXPECTED_EXIT();
9435	}
9436
9437
9438	/**
9439	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9440	*/
9441	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9442	{
9443	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9444	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9445	AssertRCReturn(rc, rc);
9446
9447	PVM pVM = pVCpu->CTX_SUFF(pVM);
9448	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9449	if (RT_LIKELY(rc == VINF_SUCCESS))
9450	{
9451	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9452	Assert(pVmxTransient->cbInstr == 2);
9453	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9454	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9455	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9456	}
9457	else
9458	{
9459	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9460	rc = VERR_EM_INTERPRETER;
9461	}
9462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9463	return rc;
9464	}
9465
9466
9467	/**
9468	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9469	*/
9470	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9471	{
9472	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9473	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9474	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9475	AssertRCReturn(rc, rc);
9476
9477	PVM pVM = pVCpu->CTX_SUFF(pVM);
9478	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9479	if (RT_LIKELY(rc == VINF_SUCCESS))
9480	{
9481	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9482	Assert(pVmxTransient->cbInstr == 3);
9483	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9484	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9485	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9486	}
9487	else
9488	{
9489	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9490	rc = VERR_EM_INTERPRETER;
9491	}
9492	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9493	return rc;
9494	}
9495
9496
9497	/**
9498	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9499	*/
9500	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9501	{
9502	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9503	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9504	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9505	AssertRCReturn(rc, rc);
9506
9507	PVM pVM = pVCpu->CTX_SUFF(pVM);
9508	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9509	if (RT_LIKELY(rc == VINF_SUCCESS))
9510	{
9511	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9512	Assert(pVmxTransient->cbInstr == 2);
9513	}
9514	else
9515	{
9516	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9517	rc = VERR_EM_INTERPRETER;
9518	}
9519	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9520	return rc;
9521	}
9522
9523
9524	/**
9525	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9526	*/
9527	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9528	{
9529	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9530	PVM pVM = pVCpu->CTX_SUFF(pVM);
9531	Assert(!pVM->hm.s.fNestedPaging);
9532
9533	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9534	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9535	AssertRCReturn(rc, rc);
9536
9537	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9538	rc = VBOXSTRICTRC_VAL(rc2);
9539	if (RT_LIKELY(rc == VINF_SUCCESS))
9540	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9541	else
9542	{
9543	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9544	pVmxTransient->uExitQualification, rc));
9545	}
9546	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9547	return rc;
9548	}
9549
9550
9551	/**
9552	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9553	*/
9554	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9555	{
9556	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9557	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9558	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9559	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9560	AssertRCReturn(rc, rc);
9561
9562	PVM pVM = pVCpu->CTX_SUFF(pVM);
9563	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9564	if (RT_LIKELY(rc == VINF_SUCCESS))
9565	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9566	else
9567	{
9568	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9569	rc = VERR_EM_INTERPRETER;
9570	}
9571	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9572	return rc;
9573	}
9574
9575
9576	/**
9577	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9578	*/
9579	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9580	{
9581	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9582	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9583	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9584	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9585	AssertRCReturn(rc, rc);
9586
9587	PVM pVM = pVCpu->CTX_SUFF(pVM);
9588	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9589	rc = VBOXSTRICTRC_VAL(rc2);
9590	if (RT_LIKELY( rc == VINF_SUCCESS
9591	\|\| rc == VINF_EM_HALT))
9592	{
9593	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9594	AssertRCReturn(rc3, rc3);
9595
9596	if ( rc == VINF_EM_HALT
9597	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9598	{
9599	rc = VINF_SUCCESS;
9600	}
9601	}
9602	else
9603	{
9604	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9605	rc = VERR_EM_INTERPRETER;
9606	}
9607	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9608	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9610	return rc;
9611	}
9612
9613
9614	/**
9615	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9616	*/
9617	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9618	{
9619	/*
9620	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9621	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9622	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9623	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9624	*/
9625	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9626	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9627	HMVMX_RETURN_UNEXPECTED_EXIT();
9628	}
9629
9630
9631	/**
9632	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9633	*/
9634	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9635	{
9636	/*
9637	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9638	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9639	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9640	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9641	*/
9642	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9643	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9644	HMVMX_RETURN_UNEXPECTED_EXIT();
9645	}
9646
9647
9648	/**
9649	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9650	*/
9651	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9652	{
9653	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9654	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9655	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9656	HMVMX_RETURN_UNEXPECTED_EXIT();
9657	}
9658
9659
9660	/**
9661	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9662	*/
9663	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9664	{
9665	/*
9666	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9667	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9668	* See Intel spec. 25.3 "Other Causes of VM-exits".
9669	*/
9670	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9671	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9672	HMVMX_RETURN_UNEXPECTED_EXIT();
9673	}
9674
9675
9676	/**
9677	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9678	* VM-exit.
9679	*/
9680	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9681	{
9682	/*
9683	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9684	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9685	*
9686	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9687	* See Intel spec. "23.8 Restrictions on VMX operation".
9688	*/
9689	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9690	return VINF_SUCCESS;
9691	}
9692
9693
9694	/**
9695	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9696	* VM-exit.
9697	*/
9698	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9699	{
9700	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9701	return VINF_EM_RESET;
9702	}
9703
9704
9705	/**
9706	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9707	*/
9708	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9709	{
9710	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9711	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9712	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9713	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9714	AssertRCReturn(rc, rc);
9715
9716	pMixedCtx->rip++;
9717	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9718	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9719	rc = VINF_SUCCESS;
9720	else
9721	rc = VINF_EM_HALT;
9722
9723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9724	return rc;
9725	}
9726
9727
9728	/**
9729	* VM-exit handler for instructions that result in a #UD exception delivered to
9730	* the guest.
9731	*/
9732	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9733	{
9734	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9735	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9736	return VINF_SUCCESS;
9737	}
9738
9739
9740	/**
9741	* VM-exit handler for expiry of the VMX preemption timer.
9742	*/
9743	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9744	{
9745	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9746
9747	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9748	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9749
9750	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9751	PVM pVM = pVCpu->CTX_SUFF(pVM);
9752	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9753	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9754	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9755	}
9756
9757
9758	/**
9759	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9760	*/
9761	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9762	{
9763	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9764
9765	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9766	/** @todo check if XSETBV is supported by the recompiler. */
9767	return VERR_EM_INTERPRETER;
9768	}
9769
9770
9771	/**
9772	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9773	*/
9774	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9775	{
9776	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9777
9778	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9779	/** @todo implement EMInterpretInvpcid() */
9780	return VERR_EM_INTERPRETER;
9781	}
9782
9783
9784	/**
9785	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9786	* Error VM-exit.
9787	*/
9788	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9789	{
9790	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9791	AssertRCReturn(rc, rc);
9792
9793	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9794	NOREF(uInvalidReason);
9795
9796	#ifdef VBOX_STRICT
9797	uint32_t uIntrState;
9798	HMVMXHCUINTREG uHCReg;
9799	uint64_t u64Val;
9800	uint32_t u32Val;
9801
9802	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9803	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9804	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9805	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9806	AssertRCReturn(rc, rc);
9807
9808	Log4(("uInvalidReason %u\n", uInvalidReason));
9809	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9810	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9811	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9812	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9813
9814	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9815	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9816	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9817	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9818	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9819	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9820	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9821	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9822	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9823	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9824	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9825	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9826	#else
9827	NOREF(pVmxTransient);
9828	#endif
9829
9830	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9831	return VERR_VMX_INVALID_GUEST_STATE;
9832	}
9833
9834
9835	/**
9836	* VM-exit handler for VM-entry failure due to an MSR-load
9837	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9838	*/
9839	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9840	{
9841	NOREF(pVmxTransient);
9842	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9843	HMVMX_RETURN_UNEXPECTED_EXIT();
9844	}
9845
9846
9847	/**
9848	* VM-exit handler for VM-entry failure due to a machine-check event
9849	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9850	*/
9851	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9852	{
9853	NOREF(pVmxTransient);
9854	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9855	HMVMX_RETURN_UNEXPECTED_EXIT();
9856	}
9857
9858
9859	/**
9860	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9861	* theory.
9862	*/
9863	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9864	{
9865	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9866	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9867	return VERR_VMX_UNDEFINED_EXIT_CODE;
9868	}
9869
9870
9871	/**
9872	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9873	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9874	* Conditional VM-exit.
9875	*/
9876	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9877	{
9878	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9879
9880	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9881	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9882	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9883	return VERR_EM_INTERPRETER;
9884	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9885	HMVMX_RETURN_UNEXPECTED_EXIT();
9886	}
9887
9888
9889	/**
9890	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9891	*/
9892	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9893	{
9894	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9895
9896	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9897	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9898	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9899	return VERR_EM_INTERPRETER;
9900	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9901	HMVMX_RETURN_UNEXPECTED_EXIT();
9902	}
9903
9904
9905	/**
9906	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9907	*/
9908	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9909	{
9910	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9911
9912	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9913	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9914	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9915	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9916	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9917	{
9918	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9919	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9920	}
9921	AssertRCReturn(rc, rc);
9922	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9923
9924	#ifdef VBOX_STRICT
9925	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9926	&& hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9927	{
9928	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9929	HMVMX_RETURN_UNEXPECTED_EXIT();
9930	}
9931	#endif
9932
9933	PVM pVM = pVCpu->CTX_SUFF(pVM);
9934	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9935	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9936	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9937	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9938
9939	if (RT_LIKELY(rc == VINF_SUCCESS))
9940	{
9941	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9942	Assert(pVmxTransient->cbInstr == 2);
9943	}
9944	return rc;
9945	}
9946
9947
9948	/**
9949	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9950	*/
9951	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9952	{
9953	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9954	PVM pVM = pVCpu->CTX_SUFF(pVM);
9955	int rc = VINF_SUCCESS;
9956
9957	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9958	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9959	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9960	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9961	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9962	{
9963	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9964	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9965	}
9966	AssertRCReturn(rc, rc);
9967	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9968
9969	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9970	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9971	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9972
9973	if (RT_LIKELY(rc == VINF_SUCCESS))
9974	{
9975	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9976
9977	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9978	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9979	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9980	{
9981	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9982	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9983	EMInterpretWrmsr() changes it. */
9984	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9985	}
9986	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9987	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9988
9989	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
9990	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9991	{
9992	switch (pMixedCtx->ecx)
9993	{
9994	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9995	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9996	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9997	case MSR_K8_FS_BASE: /* no break */
9998	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9999	default:
10000	{
10001	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10002	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10003	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10004	else if ( HMVMX_IS_64BIT_HOST_MODE()
10005	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10006	{
10007	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10008	}
10009	#endif
10010	break;
10011	}
10012	}
10013	}
10014	#ifdef VBOX_STRICT
10015	else
10016	{
10017	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10018	switch (pMixedCtx->ecx)
10019	{
10020	case MSR_IA32_SYSENTER_CS:
10021	case MSR_IA32_SYSENTER_EIP:
10022	case MSR_IA32_SYSENTER_ESP:
10023	case MSR_K8_FS_BASE:
10024	case MSR_K8_GS_BASE:
10025	{
10026	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10027	HMVMX_RETURN_UNEXPECTED_EXIT();
10028	}
10029
10030	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10031	default:
10032	{
10033	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10034	{
10035	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10036	pMixedCtx->ecx));
10037	HMVMX_RETURN_UNEXPECTED_EXIT();
10038	}
10039
10040	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10041	if ( HMVMX_IS_64BIT_HOST_MODE()
10042	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10043	{
10044	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10045	HMVMX_RETURN_UNEXPECTED_EXIT();
10046	}
10047	#endif
10048	break;
10049	}
10050	}
10051	}
10052	#endif /* VBOX_STRICT */
10053	}
10054	return rc;
10055	}
10056
10057
10058	/**
10059	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10060	*/
10061	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10062	{
10063	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10064
10065	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10066	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10067	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10068	return VERR_EM_INTERPRETER;
10069	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10070	HMVMX_RETURN_UNEXPECTED_EXIT();
10071	}
10072
10073
10074	/**
10075	* VM-exit handler for when the TPR value is lowered below the specified
10076	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10077	*/
10078	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10079	{
10080	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10081	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10082
10083	/*
10084	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10085	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10086	* resume guest execution.
10087	*/
10088	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10089	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10090	return VINF_SUCCESS;
10091	}
10092
10093
10094	/**
10095	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10096	* VM-exit.
10097	*
10098	* @retval VINF_SUCCESS when guest execution can continue.
10099	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10100	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10101	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10102	* recompiler.
10103	*/
10104	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10105	{
10106	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10107	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10108	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10109	AssertRCReturn(rc, rc);
10110
10111	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10112	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10113	PVM pVM = pVCpu->CTX_SUFF(pVM);
10114	switch (uAccessType)
10115	{
10116	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10117	{
10118	#if 0
10119	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10120	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10121	#else
10122	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10123	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10124	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10125	#endif
10126	AssertRCReturn(rc, rc);
10127
10128	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10129	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10130	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10131	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10132
10133	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10134	{
10135	case 0: /* CR0 */
10136	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10137	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10138	break;
10139	case 2: /* CR2 */
10140	/* Nothing to do here, CR2 it's not part of the VMCS. */
10141	break;
10142	case 3: /* CR3 */
10143	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10144	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10145	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10146	break;
10147	case 4: /* CR4 */
10148	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10149	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10150	break;
10151	case 8: /* CR8 */
10152	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10153	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10154	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10155	break;
10156	default:
10157	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10158	break;
10159	}
10160
10161	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10162	break;
10163	}
10164
10165	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10166	{
10167	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10168	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10169	AssertRCReturn(rc, rc);
10170	Assert( !pVM->hm.s.fNestedPaging
10171	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10172	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10173
10174	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10175	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10176	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10177
10178	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10179	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10180	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10181	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10182	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10183	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10184	break;
10185	}
10186
10187	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10188	{
10189	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10190	AssertRCReturn(rc, rc);
10191	rc = EMInterpretCLTS(pVM, pVCpu);
10192	AssertRCReturn(rc, rc);
10193	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10194	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10195	Log4(("CRX CLTS write rc=%d\n", rc));
10196	break;
10197	}
10198
10199	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10200	{
10201	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10202	AssertRCReturn(rc, rc);
10203	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10204	if (RT_LIKELY(rc == VINF_SUCCESS))
10205	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10206	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10207	Log4(("CRX LMSW write rc=%d\n", rc));
10208	break;
10209	}
10210
10211	default:
10212	{
10213	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10214	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10215	}
10216	}
10217
10218	/* Validate possible error codes. */
10219	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10220	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10221	if (RT_SUCCESS(rc))
10222	{
10223	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10224	AssertRCReturn(rc2, rc2);
10225	}
10226
10227	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10228	return rc;
10229	}
10230
10231
10232	/**
10233	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10234	* VM-exit.
10235	*/
10236	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10237	{
10238	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10239	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10240
10241	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10242	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10243	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10244	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10245	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10246	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10247	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10248	AssertRCReturn(rc2, rc2);
10249
10250	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10251	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10252	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10253	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10254	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10255	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10256	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10257
10258	/* I/O operation lookup arrays. */
10259	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10260	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10261
10262	VBOXSTRICTRC rcStrict;
10263	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10264	const uint32_t cbInstr = pVmxTransient->cbInstr;
10265	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10266	PVM pVM = pVCpu->CTX_SUFF(pVM);
10267	if (fIOString)
10268	{
10269	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10270	/*
10271	* INS/OUTS - I/O String instruction.
10272	*
10273	* Use instruction-information if available, otherwise fall back on
10274	* interpreting the instruction.
10275	*/
10276	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10277	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10278	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10279	{
10280	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10281	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10282	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10283	AssertRCReturn(rc2, rc2);
10284	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10285	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10286	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10287	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10288	if (fIOWrite)
10289	{
10290	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10291	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10292	}
10293	else
10294	{
10295	/*
10296	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10297	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10298	* See Intel Instruction spec. for "INS".
10299	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10300	*/
10301	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10302	}
10303	}
10304	else
10305	{
10306	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10307	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10308	AssertRCReturn(rc2, rc2);
10309	rcStrict = IEMExecOne(pVCpu);
10310	}
10311	/** @todo IEM needs to be setting these flags somehow. */
10312	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10313	fUpdateRipAlready = true;
10314	#else
10315	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10316	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10317	if (RT_SUCCESS(rcStrict))
10318	{
10319	if (fIOWrite)
10320	{
10321	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10322	(DISCPUMODE)pDis->uAddrMode, cbValue);
10323	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10324	}
10325	else
10326	{
10327	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10328	(DISCPUMODE)pDis->uAddrMode, cbValue);
10329	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10330	}
10331	}
10332	else
10333	{
10334	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10335	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10336	}
10337	#endif
10338	}
10339	else
10340	{
10341	/*
10342	* IN/OUT - I/O instruction.
10343	*/
10344	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10345	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10346	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10347	if (fIOWrite)
10348	{
10349	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10350	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10351	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10352	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10353	}
10354	else
10355	{
10356	uint32_t u32Result = 0;
10357	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10358	if (IOM_SUCCESS(rcStrict))
10359	{
10360	/* Save result of I/O IN instr. in AL/AX/EAX. */
10361	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10362	}
10363	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10364	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10365	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10366	}
10367	}
10368
10369	if (IOM_SUCCESS(rcStrict))
10370	{
10371	if (!fUpdateRipAlready)
10372	{
10373	pMixedCtx->rip += cbInstr;
10374	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10375	}
10376
10377	/*
10378	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10379	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10380	*/
10381	if (fIOString)
10382	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10383
10384	/*
10385	* If any I/O breakpoints are armed, we need to check if one triggered
10386	* and take appropriate action.
10387	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10388	*/
10389	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10390	AssertRCReturn(rc2, rc2);
10391
10392	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10393	* execution engines about whether hyper BPs and such are pending. */
10394	uint32_t const uDr7 = pMixedCtx->dr[7];
10395	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10396	&& X86_DR7_ANY_RW_IO(uDr7)
10397	&& (pMixedCtx->cr4 & X86_CR4_DE))
10398	\|\| DBGFBpIsHwIoArmed(pVM)))
10399	{
10400	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10401
10402	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10403	VMMRZCallRing3Disable(pVCpu);
10404	HM_DISABLE_PREEMPT_IF_NEEDED();
10405
10406	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10407
10408	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10409	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10410	{
10411	/* Raise #DB. */
10412	if (fIsGuestDbgActive)
10413	ASMSetDR6(pMixedCtx->dr[6]);
10414	if (pMixedCtx->dr[7] != uDr7)
10415	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10416
10417	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10418	}
10419	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10420	else if ( rcStrict2 != VINF_SUCCESS
10421	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10422	rcStrict = rcStrict2;
10423
10424	HM_RESTORE_PREEMPT_IF_NEEDED();
10425	VMMRZCallRing3Enable(pVCpu);
10426	}
10427	}
10428
10429	#ifdef DEBUG
10430	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10431	Assert(!fIOWrite);
10432	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10433	Assert(fIOWrite);
10434	else
10435	{
10436	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10437	* statuses, that the VMM device and some others may return. See
10438	* IOM_SUCCESS() for guidance. */
10439	AssertMsg( RT_FAILURE(rcStrict)
10440	\|\| rcStrict == VINF_SUCCESS
10441	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10442	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10443	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10444	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10445	}
10446	#endif
10447
10448	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10449	return VBOXSTRICTRC_TODO(rcStrict);
10450	}
10451
10452
10453	/**
10454	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10455	* VM-exit.
10456	*/
10457	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10458	{
10459	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10460
10461	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10462	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10463	AssertRCReturn(rc, rc);
10464	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10465	{
10466	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10467	AssertRCReturn(rc, rc);
10468	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10469	{
10470	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10471
10472	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10473	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10474	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10475	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10476	{
10477	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10478	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10479
10480	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10481	Assert(!pVCpu->hm.s.Event.fPending);
10482	pVCpu->hm.s.Event.fPending = true;
10483	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10484	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10485	AssertRCReturn(rc, rc);
10486	if (fErrorCodeValid)
10487	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10488	else
10489	pVCpu->hm.s.Event.u32ErrCode = 0;
10490	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10491	&& uVector == X86_XCPT_PF)
10492	{
10493	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10494	}
10495
10496	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10497	}
10498	}
10499	}
10500
10501	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10502	* emulation. */
10503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10504	return VERR_EM_INTERPRETER;
10505	}
10506
10507
10508	/**
10509	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10510	*/
10511	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10512	{
10513	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10514	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10515	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10516	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10517	AssertRCReturn(rc, rc);
10518	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10519	return VINF_EM_DBG_STEPPED;
10520	}
10521
10522
10523	/**
10524	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10525	*/
10526	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10527	{
10528	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10529
10530	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10531	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10532	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10533	return VINF_SUCCESS;
10534	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10535	return rc;
10536
10537	#if 0
10538	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10539	* just sync the whole thing. */
10540	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10541	#else
10542	/* Aggressive state sync. for now. */
10543	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10544	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10545	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10546	#endif
10547	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10548	AssertRCReturn(rc, rc);
10549
10550	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10551	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10552	switch (uAccessType)
10553	{
10554	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10555	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10556	{
10557	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10558	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10559	{
10560	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10561	}
10562
10563	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10564	GCPhys &= PAGE_BASE_GC_MASK;
10565	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10566	PVM pVM = pVCpu->CTX_SUFF(pVM);
10567	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10568	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10569
10570	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10571	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10572	CPUMCTX2CORE(pMixedCtx), GCPhys);
10573	rc = VBOXSTRICTRC_VAL(rc2);
10574	Log4(("ApicAccess rc=%d\n", rc));
10575	if ( rc == VINF_SUCCESS
10576	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10577	\|\| rc == VERR_PAGE_NOT_PRESENT)
10578	{
10579	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10580	\| HM_CHANGED_GUEST_RSP
10581	\| HM_CHANGED_GUEST_RFLAGS
10582	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10583	rc = VINF_SUCCESS;
10584	}
10585	break;
10586	}
10587
10588	default:
10589	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10590	rc = VINF_EM_RAW_EMULATE_INSTR;
10591	break;
10592	}
10593
10594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10595	return rc;
10596	}
10597
10598
10599	/**
10600	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10601	* VM-exit.
10602	*/
10603	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10604	{
10605	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10606
10607	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10608	if (pVmxTransient->fWasGuestDebugStateActive)
10609	{
10610	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10611	HMVMX_RETURN_UNEXPECTED_EXIT();
10612	}
10613
10614	int rc = VERR_INTERNAL_ERROR_5;
10615	if ( !DBGFIsStepping(pVCpu)
10616	&& !pVCpu->hm.s.fSingleInstruction
10617	&& !pVmxTransient->fWasHyperDebugStateActive)
10618	{
10619	/* Don't intercept MOV DRx and #DB any more. */
10620	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10621	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10622	AssertRCReturn(rc, rc);
10623
10624	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10625	{
10626	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10627	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10628	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10629	AssertRCReturn(rc, rc);
10630	#endif
10631	}
10632
10633	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10634	VMMRZCallRing3Disable(pVCpu);
10635	HM_DISABLE_PREEMPT_IF_NEEDED();
10636
10637	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10638	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10639	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10640
10641	HM_RESTORE_PREEMPT_IF_NEEDED();
10642	VMMRZCallRing3Enable(pVCpu);
10643
10644	#ifdef VBOX_WITH_STATISTICS
10645	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10646	AssertRCReturn(rc, rc);
10647	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10648	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10649	else
10650	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10651	#endif
10652	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10653	return VINF_SUCCESS;
10654	}
10655
10656	/*
10657	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10658	* Update the segment registers and DR7 from the CPU.
10659	*/
10660	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10661	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10662	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10663	AssertRCReturn(rc, rc);
10664	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10665
10666	PVM pVM = pVCpu->CTX_SUFF(pVM);
10667	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10668	{
10669	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10670	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10671	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10672	if (RT_SUCCESS(rc))
10673	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10674	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10675	}
10676	else
10677	{
10678	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10679	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10680	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10681	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10682	}
10683
10684	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10685	if (RT_SUCCESS(rc))
10686	{
10687	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10688	AssertRCReturn(rc2, rc2);
10689	}
10690	return rc;
10691	}
10692
10693
10694	/**
10695	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10696	* Conditional VM-exit.
10697	*/
10698	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10699	{
10700	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10701	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10702
10703	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10704	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10705	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10706	return VINF_SUCCESS;
10707	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10708	return rc;
10709
10710	RTGCPHYS GCPhys = 0;
10711	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10712
10713	#if 0
10714	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10715	#else
10716	/* Aggressive state sync. for now. */
10717	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10718	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10719	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10720	#endif
10721	AssertRCReturn(rc, rc);
10722
10723	/*
10724	* If we succeed, resume guest execution.
10725	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10726	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10727	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10728	* weird case. See @bugref{6043}.
10729	*/
10730	PVM pVM = pVCpu->CTX_SUFF(pVM);
10731	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10732	rc = VBOXSTRICTRC_VAL(rc2);
10733	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10734	if ( rc == VINF_SUCCESS
10735	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10736	\|\| rc == VERR_PAGE_NOT_PRESENT)
10737	{
10738	/* Successfully handled MMIO operation. */
10739	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10740	\| HM_CHANGED_GUEST_RSP
10741	\| HM_CHANGED_GUEST_RFLAGS
10742	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10743	rc = VINF_SUCCESS;
10744	}
10745	return rc;
10746	}
10747
10748
10749	/**
10750	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10751	* VM-exit.
10752	*/
10753	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10754	{
10755	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10756	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10757
10758	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10759	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10760	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10761	return VINF_SUCCESS;
10762	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10763	return rc;
10764
10765	RTGCPHYS GCPhys = 0;
10766	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10767	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10768	#if 0
10769	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10770	#else
10771	/* Aggressive state sync. for now. */
10772	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10773	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10774	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10775	#endif
10776	AssertRCReturn(rc, rc);
10777
10778	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10779	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10780
10781	RTGCUINT uErrorCode = 0;
10782	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10783	uErrorCode \|= X86_TRAP_PF_ID;
10784	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10785	uErrorCode \|= X86_TRAP_PF_RW;
10786	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10787	uErrorCode \|= X86_TRAP_PF_P;
10788
10789	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10790
10791	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10792	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10793
10794	/* Handle the pagefault trap for the nested shadow table. */
10795	PVM pVM = pVCpu->CTX_SUFF(pVM);
10796	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10797	TRPMResetTrap(pVCpu);
10798
10799	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10800	if ( rc == VINF_SUCCESS
10801	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10802	\|\| rc == VERR_PAGE_NOT_PRESENT)
10803	{
10804	/* Successfully synced our nested page tables. */
10805	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10806	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10807	\| HM_CHANGED_GUEST_RSP
10808	\| HM_CHANGED_GUEST_RFLAGS);
10809	return VINF_SUCCESS;
10810	}
10811
10812	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10813	return rc;
10814	}
10815
10816	/** @} */
10817
10818	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10819	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10820	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10821
10822	/** @name VM-exit exception handlers.
10823	* @{
10824	*/
10825
10826	/**
10827	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10828	*/
10829	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10830	{
10831	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10832	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10833
10834	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10835	AssertRCReturn(rc, rc);
10836
10837	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10838	{
10839	/* Old-style FPU error reporting needs some extra work. */
10840	/** @todo don't fall back to the recompiler, but do it manually. */
10841	return VERR_EM_INTERPRETER;
10842	}
10843
10844	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10845	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10846	return rc;
10847	}
10848
10849
10850	/**
10851	* VM-exit exception handler for #BP (Breakpoint exception).
10852	*/
10853	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10854	{
10855	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10857
10858	/** @todo Try optimize this by not saving the entire guest state unless
10859	* really needed. */
10860	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10861	AssertRCReturn(rc, rc);
10862
10863	PVM pVM = pVCpu->CTX_SUFF(pVM);
10864	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10865	if (rc == VINF_EM_RAW_GUEST_TRAP)
10866	{
10867	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10868	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10869	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10870	AssertRCReturn(rc, rc);
10871
10872	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10873	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10874	}
10875
10876	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10877	return rc;
10878	}
10879
10880
10881	/**
10882	* VM-exit exception handler for #DB (Debug exception).
10883	*/
10884	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10885	{
10886	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10887	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10888	Log6(("XcptDB\n"));
10889
10890	/*
10891	* Get the DR6-like values from the exit qualification and pass it to DBGF
10892	* for processing.
10893	*/
10894	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10895	AssertRCReturn(rc, rc);
10896
10897	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10898	uint64_t uDR6 = X86_DR6_INIT_VAL;
10899	uDR6 \|= ( pVmxTransient->uExitQualification
10900	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10901
10902	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10903	if (rc == VINF_EM_RAW_GUEST_TRAP)
10904	{
10905	/*
10906	* The exception was for the guest. Update DR6, DR7.GD and
10907	* IA32_DEBUGCTL.LBR before forwarding it.
10908	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10909	*/
10910	VMMRZCallRing3Disable(pVCpu);
10911	HM_DISABLE_PREEMPT_IF_NEEDED();
10912
10913	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10914	pMixedCtx->dr[6] \|= uDR6;
10915	if (CPUMIsGuestDebugStateActive(pVCpu))
10916	ASMSetDR6(pMixedCtx->dr[6]);
10917
10918	HM_RESTORE_PREEMPT_IF_NEEDED();
10919	VMMRZCallRing3Enable(pVCpu);
10920
10921	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10922	AssertRCReturn(rc, rc);
10923
10924	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10925	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10926
10927	/* Paranoia. */
10928	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10929	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10930
10931	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10932	AssertRCReturn(rc, rc);
10933
10934	/*
10935	* Raise #DB in the guest.
10936	*/
10937	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10938	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10939	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10940	AssertRCReturn(rc, rc);
10941	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10942	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10943	return VINF_SUCCESS;
10944	}
10945
10946	/*
10947	* Not a guest trap, must be a hypervisor related debug event then.
10948	* Update DR6 in case someone is interested in it.
10949	*/
10950	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10951	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10952	CPUMSetHyperDR6(pVCpu, uDR6);
10953
10954	return rc;
10955	}
10956
10957
10958	/**
10959	* VM-exit exception handler for #NM (Device-not-available exception: floating
10960	* point exception).
10961	*/
10962	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10963	{
10964	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10965
10966	/* We require CR0 and EFER. EFER is always up-to-date. */
10967	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10968	AssertRCReturn(rc, rc);
10969
10970	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10971	VMMRZCallRing3Disable(pVCpu);
10972	HM_DISABLE_PREEMPT_IF_NEEDED();
10973
10974	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10975	if (pVmxTransient->fWasGuestFPUStateActive)
10976	{
10977	rc = VINF_EM_RAW_GUEST_TRAP;
10978	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10979	}
10980	else
10981	{
10982	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10983	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10984	#endif
10985	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10986	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10987	}
10988
10989	HM_RESTORE_PREEMPT_IF_NEEDED();
10990	VMMRZCallRing3Enable(pVCpu);
10991
10992	if (rc == VINF_SUCCESS)
10993	{
10994	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10995	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10996	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10997	pVCpu->hm.s.fUseGuestFpu = true;
10998	}
10999	else
11000	{
11001	/* Forward #NM to the guest. */
11002	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11003	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11004	AssertRCReturn(rc, rc);
11005	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11006	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11008	}
11009
11010	return VINF_SUCCESS;
11011	}
11012
11013
11014	/**
11015	* VM-exit exception handler for #GP (General-protection exception).
11016	*
11017	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11018	*/
11019	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11020	{
11021	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11022	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11023
11024	int rc = VERR_INTERNAL_ERROR_5;
11025	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11026	{
11027	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11028	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11029	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11030	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11031	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11032	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11033	AssertRCReturn(rc, rc);
11034	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11035	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11036	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11037	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11038	return rc;
11039	#else
11040	/* We don't intercept #GP. */
11041	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11042	NOREF(pVmxTransient);
11043	return VERR_VMX_UNEXPECTED_EXCEPTION;
11044	#endif
11045	}
11046
11047	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11048	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11049
11050	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11051	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11052	AssertRCReturn(rc, rc);
11053
11054	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11055	uint32_t cbOp = 0;
11056	PVM pVM = pVCpu->CTX_SUFF(pVM);
11057	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11058	if (RT_SUCCESS(rc))
11059	{
11060	rc = VINF_SUCCESS;
11061	Assert(cbOp == pDis->cbInstr);
11062	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11063	switch (pDis->pCurInstr->uOpcode)
11064	{
11065	case OP_CLI:
11066	{
11067	pMixedCtx->eflags.Bits.u1IF = 0;
11068	pMixedCtx->rip += pDis->cbInstr;
11069	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11070	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11071	break;
11072	}
11073
11074	case OP_STI:
11075	{
11076	pMixedCtx->eflags.Bits.u1IF = 1;
11077	pMixedCtx->rip += pDis->cbInstr;
11078	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11079	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11080	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11081	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11082	break;
11083	}
11084
11085	case OP_HLT:
11086	{
11087	rc = VINF_EM_HALT;
11088	pMixedCtx->rip += pDis->cbInstr;
11089	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11091	break;
11092	}
11093
11094	case OP_POPF:
11095	{
11096	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11097	uint32_t cbParm = 0;
11098	uint32_t uMask = 0;
11099	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11100	{
11101	cbParm = 4;
11102	uMask = 0xffffffff;
11103	}
11104	else
11105	{
11106	cbParm = 2;
11107	uMask = 0xffff;
11108	}
11109
11110	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11111	RTGCPTR GCPtrStack = 0;
11112	X86EFLAGS Eflags;
11113	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11114	&GCPtrStack);
11115	if (RT_SUCCESS(rc))
11116	{
11117	Assert(sizeof(Eflags.u32) >= cbParm);
11118	Eflags.u32 = 0;
11119	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11120	}
11121	if (RT_FAILURE(rc))
11122	{
11123	rc = VERR_EM_INTERPRETER;
11124	break;
11125	}
11126	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11127	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11128	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11129	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11130	pMixedCtx->esp += cbParm;
11131	pMixedCtx->esp &= uMask;
11132	pMixedCtx->rip += pDis->cbInstr;
11133
11134	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11135	\| HM_CHANGED_GUEST_RSP
11136	\| HM_CHANGED_GUEST_RFLAGS);
11137	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11138	break;
11139	}
11140
11141	case OP_PUSHF:
11142	{
11143	uint32_t cbParm = 0;
11144	uint32_t uMask = 0;
11145	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11146	{
11147	cbParm = 4;
11148	uMask = 0xffffffff;
11149	}
11150	else
11151	{
11152	cbParm = 2;
11153	uMask = 0xffff;
11154	}
11155
11156	/* Get the stack pointer & push the contents of eflags onto the stack. */
11157	RTGCPTR GCPtrStack = 0;
11158	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11159	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11160	if (RT_FAILURE(rc))
11161	{
11162	rc = VERR_EM_INTERPRETER;
11163	break;
11164	}
11165	X86EFLAGS Eflags = pMixedCtx->eflags;
11166	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11167	Eflags.Bits.u1RF = 0;
11168	Eflags.Bits.u1VM = 0;
11169
11170	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11171	if (RT_FAILURE(rc))
11172	{
11173	rc = VERR_EM_INTERPRETER;
11174	break;
11175	}
11176	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11177	pMixedCtx->esp -= cbParm;
11178	pMixedCtx->esp &= uMask;
11179	pMixedCtx->rip += pDis->cbInstr;
11180	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11181	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11182	break;
11183	}
11184
11185	case OP_IRET:
11186	{
11187	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11188	* instruction reference. */
11189	RTGCPTR GCPtrStack = 0;
11190	uint32_t uMask = 0xffff;
11191	uint16_t aIretFrame[3];
11192	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11193	{
11194	rc = VERR_EM_INTERPRETER;
11195	break;
11196	}
11197	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11198	&GCPtrStack);
11199	if (RT_SUCCESS(rc))
11200	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11201	if (RT_FAILURE(rc))
11202	{
11203	rc = VERR_EM_INTERPRETER;
11204	break;
11205	}
11206	pMixedCtx->eip = 0;
11207	pMixedCtx->ip = aIretFrame[0];
11208	pMixedCtx->cs.Sel = aIretFrame[1];
11209	pMixedCtx->cs.ValidSel = aIretFrame[1];
11210	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11211	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11212	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11213	pMixedCtx->sp += sizeof(aIretFrame);
11214	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11215	\| HM_CHANGED_GUEST_SEGMENT_REGS
11216	\| HM_CHANGED_GUEST_RSP
11217	\| HM_CHANGED_GUEST_RFLAGS);
11218	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11219	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11220	break;
11221	}
11222
11223	case OP_INT:
11224	{
11225	uint16_t uVector = pDis->Param1.uValue & 0xff;
11226	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11228	break;
11229	}
11230
11231	case OP_INTO:
11232	{
11233	if (pMixedCtx->eflags.Bits.u1OF)
11234	{
11235	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11236	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11237	}
11238	break;
11239	}
11240
11241	default:
11242	{
11243	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11244	EMCODETYPE_SUPERVISOR);
11245	rc = VBOXSTRICTRC_VAL(rc2);
11246	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11247	Log4(("#GP rc=%Rrc\n", rc));
11248	break;
11249	}
11250	}
11251	}
11252	else
11253	rc = VERR_EM_INTERPRETER;
11254
11255	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11256	("#GP Unexpected rc=%Rrc\n", rc));
11257	return rc;
11258	}
11259
11260
11261	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11262	/**
11263	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11264	* the exception reported in the VMX transient structure back into the VM.
11265	*
11266	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11267	* up-to-date.
11268	*/
11269	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11270	{
11271	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11272
11273	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11274	hmR0VmxCheckExitDueToEventDelivery(). */
11275	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11276	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11277	AssertRCReturn(rc, rc);
11278	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11279
11280	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11281	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11282	return VINF_SUCCESS;
11283	}
11284	#endif
11285
11286
11287	/**
11288	* VM-exit exception handler for #PF (Page-fault exception).
11289	*/
11290	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11291	{
11292	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11293	PVM pVM = pVCpu->CTX_SUFF(pVM);
11294	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11295	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11296	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11297	AssertRCReturn(rc, rc);
11298
11299	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11300	if (pVM->hm.s.fNestedPaging)
11301	{
11302	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11303	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11304	{
11305	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11306	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11307	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11308	}
11309	else
11310	{
11311	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11312	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11313	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11314	}
11315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11316	return rc;
11317	}
11318	#else
11319	Assert(!pVM->hm.s.fNestedPaging);
11320	NOREF(pVM);
11321	#endif
11322
11323	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11324	AssertRCReturn(rc, rc);
11325
11326	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11327	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11328
11329	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11330	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11331	(RTGCPTR)pVmxTransient->uExitQualification);
11332
11333	Log4(("#PF: rc=%Rrc\n", rc));
11334	if (rc == VINF_SUCCESS)
11335	{
11336	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11337	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11338	* memory? We don't update the whole state here... */
11339	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11340	\| HM_CHANGED_GUEST_RSP
11341	\| HM_CHANGED_GUEST_RFLAGS
11342	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11343	TRPMResetTrap(pVCpu);
11344	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11345	return rc;
11346	}
11347	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11348	{
11349	if (!pVmxTransient->fVectoringPF)
11350	{
11351	/* It's a guest page fault and needs to be reflected to the guest. */
11352	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11353	TRPMResetTrap(pVCpu);
11354	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11355	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11356	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11357	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11358	}
11359	else
11360	{
11361	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11362	TRPMResetTrap(pVCpu);
11363	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11364	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11365	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11366	}
11367
11368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11369	return VINF_SUCCESS;
11370	}
11371
11372	TRPMResetTrap(pVCpu);
11373	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11374	return rc;
11375	}
11376
11377	/** @} */
11378

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49664

Download in other formats: