HMVMXR0.cpp@ 49721

Last change on this file since 49721 was 49717, checked in by vboxsync, 11 years ago
HMVMX: Do not fall back on FERR emulation (see #6117).
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 470.7 KB

Line
1	/* $Id: HMVMXR0.cpp 49717 2013-11-29 10:13:51Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu Pointer to the VMCPU.
1240	* @param uMsr The MSR.
1241	* @param uGuestMsr Value of the guest MSR.
1242	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1243	* necessary.
1244	*/
1245	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1246	{
1247	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1248	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1249	uint32_t i;
1250	for (i = 0; i < cMsrs; i++)
1251	{
1252	if (pGuestMsr->u32Msr == uMsr)
1253	break;
1254	pGuestMsr++;
1255	}
1256
1257	bool fAdded = false;
1258	if (i == cMsrs)
1259	{
1260	++cMsrs;
1261	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1262	AssertRCReturn(rc, rc);
1263
1264	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1265	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1266	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1267
1268	fAdded = true;
1269	}
1270
1271	/* Update the MSR values in the auto-load/store MSR area. */
1272	pGuestMsr->u32Msr = uMsr;
1273	pGuestMsr->u64Value = uGuestMsrValue;
1274
1275	/* Create/update the MSR slot in the host MSR area. */
1276	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1277	pHostMsr += i;
1278	pHostMsr->u32Msr = uMsr;
1279
1280	/*
1281	* Update the host MSR only when requested by the caller AND when we're
1282	* adding it to the auto-load/store area. Otherwise, it would have been
1283	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1284	*/
1285	if ( fAdded
1286	&& fUpdateHostMsr)
1287	{
1288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1290	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1291	}
1292
1293	return VINF_SUCCESS;
1294	}
1295
1296
1297	/**
1298	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1299	* auto-load/store MSR area in the VMCS.
1300	*
1301	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1302	* area.
1303	*
1304	* @returns VBox status code.
1305	* @param pVCpu Pointer to the VMCPU.
1306	* @param uMsr The MSR.
1307	*/
1308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1309	{
1310	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1311	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1312	for (uint32_t i = 0; i < cMsrs; i++)
1313	{
1314	/* Find the MSR. */
1315	if (pGuestMsr->u32Msr == uMsr)
1316	{
1317	/* If it's the last MSR, simply reduce the count. */
1318	if (i == cMsrs - 1)
1319	{
1320	--cMsrs;
1321	break;
1322	}
1323
1324	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1325	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326	pLastGuestMsr += cMsrs;
1327	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1328	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1329
1330	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332	pLastHostMsr += cMsrs;
1333	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1334	pHostMsr->u64Value = pLastHostMsr->u64Value;
1335	--cMsrs;
1336	break;
1337	}
1338	pGuestMsr++;
1339	}
1340
1341	/* Update the VMCS if the count changed (meaning the MSR was found). */
1342	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1343	{
1344	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1345	AssertRCReturn(rc, rc);
1346
1347	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1348	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1349	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1350	}
1351
1352	return VINF_SUCCESS;
1353	}
1354
1355
1356	/**
1357	* Checks if the specified guest MSR is part of the auto-load/store area in
1358	* the VMCS.
1359	*
1360	* @returns true if found, false otherwise.
1361	* @param pVCpu Pointer to the VMCPU.
1362	* @param uMsr The MSR to find.
1363	*/
1364	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1365	{
1366	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368
1369	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1370	{
1371	if (pGuestMsr->u32Msr == uMsr)
1372	return true;
1373	}
1374	return false;
1375	}
1376
1377
1378	/**
1379	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1380	*
1381	* @param pVCpu Pointer to the VMCPU.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1386	{
1387	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1388	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1389	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1390	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1391
1392	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1393	{
1394	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1395	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1396	}
1397
1398	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1399	}
1400
1401
1402	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1403	/**
1404	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1405	* perform lazy restoration of the host MSRs while leaving VT-x.
1406	*
1407	* @param pVCpu Pointer to the VMCPU.
1408	*
1409	* @remarks No-long-jump zone!!!
1410	*/
1411	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1412	{
1413	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1414
1415	/*
1416	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1417	*/
1418	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1419	{
1420	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1421	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1422	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1423	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1424	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1425	}
1426	}
1427
1428
1429	/**
1430	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1431	* lazily while leaving VT-x.
1432	*
1433	* @returns true if it does, false otherwise.
1434	* @param pVCpu Pointer to the VMCPU.
1435	* @param uMsr The MSR to check.
1436	*/
1437	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1438	{
1439	NOREF(pVCpu);
1440	switch (uMsr)
1441	{
1442	case MSR_K8_LSTAR:
1443	case MSR_K6_STAR:
1444	case MSR_K8_SF_MASK:
1445	case MSR_K8_KERNEL_GS_BASE:
1446	return true;
1447	}
1448	return false;
1449	}
1450
1451
1452	/**
1453	* Saves a set of guests MSRs back into the guest-CPU context.
1454	*
1455	* @param pVCpu Pointer to the VMCPU.
1456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1457	* out-of-sync. Make sure to update the required fields
1458	* before using them.
1459	*
1460	* @remarks No-long-jump zone!!!
1461	*/
1462	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1463	{
1464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1465	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1466
1467	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1468	{
1469	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1470	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1471	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1472	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1473	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1474	}
1475	}
1476
1477
1478	/**
1479	* Loads a set of guests MSRs to allow read/passthru to the guest.
1480	*
1481	* The name of this function is slightly confusing. This function does NOT
1482	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1483	* common prefix for functions dealing with "lazy restoration" of the shared
1484	* MSRs.
1485	*
1486	* @param pVCpu Pointer to the VMCPU.
1487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1488	* out-of-sync. Make sure to update the required fields
1489	* before using them.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1497
1498	#if 0 /* Disabled until issue with non-atomic flag updates is resolved. See @bugref{6398#c170}. */
1499	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1500	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1501	{
1502	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1503	do { \
1504	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1505	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1506	else \
1507	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1508	} while (0)
1509
1510	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1511	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1513	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1514	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1515	}
1516	else
1517	#endif
1518	{
1519	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1520	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1521	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1522	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1523	}
1524	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1525	}
1526
1527
1528	/**
1529	* Performs lazy restoration of the set of host MSRs if they were previously
1530	* loaded with guest MSR values.
1531	*
1532	* @param pVCpu Pointer to the VMCPU.
1533	*
1534	* @remarks No-long-jump zone!!!
1535	* @remarks The guest MSRs should have been saved back into the guest-CPU
1536	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1537	*/
1538	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1539	{
1540	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1541	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1542
1543	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1544	{
1545	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1546	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1547	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1548	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1549	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1550	}
1551	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1552	}
1553	#endif /* HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
1554
1555
1556	#ifdef VBOX_STRICT
1557	/**
1558	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1559	* VMCS are correct.
1560	*
1561	* @param pVCpu Pointer to the VMCPU.
1562	*/
1563	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1564	{
1565	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1566
1567	/* Verify MSR counts in the VMCS are what we think it should be. */
1568	uint32_t cMsrs;
1569	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1570	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1571
1572	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1573	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1574
1575	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1576	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1577
1578	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1579	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1580	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1581	{
1582	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1583	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1584	pGuestMsr->u32Msr));
1585
1586	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1587	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1588	pHostMsr->u64Value, u64Msr));
1589
1590	/* Verify that the permissions are as expected in the MSR bitmap. */
1591	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1592	{
1593	VMXMSREXITREAD enmRead;
1594	VMXMSREXITWRITE enmWrite;
1595	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1596	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1597	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1598	pGuestMsr->u32Msr));
1599	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1600	pGuestMsr->u32Msr));
1601	}
1602	}
1603	}
1604	# endif /* VBOX_STRICT */
1605
1606
1607	/**
1608	* Flushes the TLB using EPT.
1609	*
1610	* @returns VBox status code.
1611	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1612	* enmFlush).
1613	* @param enmFlush Type of flush.
1614	*
1615	* @remarks Caller is responsible for making sure this function is called only
1616	* when NestedPaging is supported and providing @a enmFlush that is
1617	* supported by the CPU.
1618	* @remarks Can be called with interrupts disabled.
1619	*/
1620	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1621	{
1622	uint64_t au64Descriptor[2];
1623	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1624	au64Descriptor[0] = 0;
1625	else
1626	{
1627	Assert(pVCpu);
1628	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1629	}
1630	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1631
1632	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1633	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1634	rc));
1635	if ( RT_SUCCESS(rc)
1636	&& pVCpu)
1637	{
1638	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1639	}
1640	}
1641
1642
1643	/**
1644	* Flushes the TLB using VPID.
1645	*
1646	* @returns VBox status code.
1647	* @param pVM Pointer to the VM.
1648	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1649	* enmFlush).
1650	* @param enmFlush Type of flush.
1651	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1652	* on @a enmFlush).
1653	*
1654	* @remarks Can be called with interrupts disabled.
1655	*/
1656	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1657	{
1658	NOREF(pVM);
1659	AssertPtr(pVM);
1660	Assert(pVM->hm.s.vmx.fVpid);
1661
1662	uint64_t au64Descriptor[2];
1663	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1664	{
1665	au64Descriptor[0] = 0;
1666	au64Descriptor[1] = 0;
1667	}
1668	else
1669	{
1670	AssertPtr(pVCpu);
1671	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1672	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1673	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1674	au64Descriptor[1] = GCPtr;
1675	}
1676
1677	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1678	AssertMsg(rc == VINF_SUCCESS,
1679	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1680	if ( RT_SUCCESS(rc)
1681	&& pVCpu)
1682	{
1683	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1684	}
1685	}
1686
1687
1688	/**
1689	* Invalidates a guest page by guest virtual address. Only relevant for
1690	* EPT/VPID, otherwise there is nothing really to invalidate.
1691	*
1692	* @returns VBox status code.
1693	* @param pVM Pointer to the VM.
1694	* @param pVCpu Pointer to the VMCPU.
1695	* @param GCVirt Guest virtual address of the page to invalidate.
1696	*/
1697	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1698	{
1699	AssertPtr(pVM);
1700	AssertPtr(pVCpu);
1701	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1702
1703	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1704	if (!fFlushPending)
1705	{
1706	/*
1707	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1708	* See @bugref{6043} and @bugref{6177}.
1709	*
1710	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1711	* function maybe called in a loop with individual addresses.
1712	*/
1713	if (pVM->hm.s.vmx.fVpid)
1714	{
1715	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1716	{
1717	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1718	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1719	}
1720	else
1721	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1722	}
1723	else if (pVM->hm.s.fNestedPaging)
1724	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1725	}
1726
1727	return VINF_SUCCESS;
1728	}
1729
1730
1731	/**
1732	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1733	* otherwise there is nothing really to invalidate.
1734	*
1735	* @returns VBox status code.
1736	* @param pVM Pointer to the VM.
1737	* @param pVCpu Pointer to the VMCPU.
1738	* @param GCPhys Guest physical address of the page to invalidate.
1739	*/
1740	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1741	{
1742	NOREF(pVM); NOREF(GCPhys);
1743	LogFlowFunc(("%RGp\n", GCPhys));
1744
1745	/*
1746	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1747	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1748	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1749	*/
1750	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1751	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1752	return VINF_SUCCESS;
1753	}
1754
1755
1756	/**
1757	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1758	* case where neither EPT nor VPID is supported by the CPU.
1759	*
1760	* @param pVM Pointer to the VM.
1761	* @param pVCpu Pointer to the VMCPU.
1762	* @param pCpu Pointer to the global HM struct.
1763	*
1764	* @remarks Called with interrupts disabled.
1765	*/
1766	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1767	{
1768	AssertPtr(pVCpu);
1769	AssertPtr(pCpu);
1770	NOREF(pVM);
1771
1772	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1773
1774	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1775	#if 0
1776	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1777	pVCpu->hm.s.TlbShootdown.cPages = 0;
1778	#endif
1779
1780	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1781	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1782	pVCpu->hm.s.fForceTLBFlush = false;
1783	return;
1784	}
1785
1786
1787	/**
1788	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1789	*
1790	* @param pVM Pointer to the VM.
1791	* @param pVCpu Pointer to the VMCPU.
1792	* @param pCpu Pointer to the global HM CPU struct.
1793	* @remarks All references to "ASID" in this function pertains to "VPID" in
1794	* Intel's nomenclature. The reason is, to avoid confusion in compare
1795	* statements since the host-CPU copies are named "ASID".
1796	*
1797	* @remarks Called with interrupts disabled.
1798	*/
1799	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1800	{
1801	#ifdef VBOX_WITH_STATISTICS
1802	bool fTlbFlushed = false;
1803	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1804	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1805	if (!fTlbFlushed) \
1806	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1807	} while (0)
1808	#else
1809	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1810	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1811	#endif
1812
1813	AssertPtr(pVM);
1814	AssertPtr(pCpu);
1815	AssertPtr(pVCpu);
1816	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1817	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1818	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1819
1820
1821	/*
1822	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1823	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1824	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1825	*/
1826	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1827	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1828	{
1829	++pCpu->uCurrentAsid;
1830	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1831	{
1832	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1833	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1834	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1835	}
1836
1837	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1838	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1839	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1840
1841	/*
1842	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1843	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1844	*/
1845	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1846	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1847	HMVMX_SET_TAGGED_TLB_FLUSHED();
1848	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1849	}
1850
1851	/* Check for explicit TLB shootdowns. */
1852	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1853	{
1854	/*
1855	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1856	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1857	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1858	* but not guest-physical mappings.
1859	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1860	*/
1861	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1862	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1863	HMVMX_SET_TAGGED_TLB_FLUSHED();
1864	}
1865
1866	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1867	* where it is commented out. Support individual entry flushing
1868	* someday. */
1869	#if 0
1870	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1871	{
1872	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1873
1874	/*
1875	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1876	* as supported by the CPU.
1877	*/
1878	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1879	{
1880	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1881	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1882	}
1883	else
1884	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1885
1886	HMVMX_SET_TAGGED_TLB_FLUSHED();
1887	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1888	pVCpu->hm.s.TlbShootdown.cPages = 0;
1889	}
1890	#endif
1891
1892	pVCpu->hm.s.fForceTLBFlush = false;
1893
1894	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1895
1896	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1897	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1898	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1899	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1900	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1901	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1902	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1903	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1904
1905	/* Update VMCS with the VPID. */
1906	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1907	AssertRC(rc);
1908
1909	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1910	}
1911
1912
1913	/**
1914	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1915	*
1916	* @returns VBox status code.
1917	* @param pVM Pointer to the VM.
1918	* @param pVCpu Pointer to the VMCPU.
1919	* @param pCpu Pointer to the global HM CPU struct.
1920	*
1921	* @remarks Called with interrupts disabled.
1922	*/
1923	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1924	{
1925	AssertPtr(pVM);
1926	AssertPtr(pVCpu);
1927	AssertPtr(pCpu);
1928	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1929	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1930
1931	/*
1932	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1933	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1934	*/
1935	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1936	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1937	{
1938	pVCpu->hm.s.fForceTLBFlush = true;
1939	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1940	}
1941
1942	/* Check for explicit TLB shootdown flushes. */
1943	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1944	{
1945	pVCpu->hm.s.fForceTLBFlush = true;
1946	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1947	}
1948
1949	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1950	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1951
1952	if (pVCpu->hm.s.fForceTLBFlush)
1953	{
1954	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1955	pVCpu->hm.s.fForceTLBFlush = false;
1956	}
1957	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1958	* where it is commented out. Support individual entry flushing
1959	* someday. */
1960	#if 0
1961	else
1962	{
1963	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1964	{
1965	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1966	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1967	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1968	}
1969	else
1970	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1971
1972	pVCpu->hm.s.TlbShootdown.cPages = 0;
1973	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1974	}
1975	#endif
1976	}
1977
1978
1979	/**
1980	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1981	*
1982	* @returns VBox status code.
1983	* @param pVM Pointer to the VM.
1984	* @param pVCpu Pointer to the VMCPU.
1985	* @param pCpu Pointer to the global HM CPU struct.
1986	*
1987	* @remarks Called with interrupts disabled.
1988	*/
1989	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1990	{
1991	AssertPtr(pVM);
1992	AssertPtr(pVCpu);
1993	AssertPtr(pCpu);
1994	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1995	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1996
1997	/*
1998	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1999	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2000	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2001	*/
2002	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2003	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2004	{
2005	pVCpu->hm.s.fForceTLBFlush = true;
2006	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2007	}
2008
2009	/* Check for explicit TLB shootdown flushes. */
2010	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2011	{
2012	/*
2013	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2014	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2015	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2016	*/
2017	pVCpu->hm.s.fForceTLBFlush = true;
2018	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2019	}
2020
2021	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2022	if (pVCpu->hm.s.fForceTLBFlush)
2023	{
2024	++pCpu->uCurrentAsid;
2025	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2026	{
2027	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2028	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2029	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2030	}
2031
2032	pVCpu->hm.s.fForceTLBFlush = false;
2033	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2034	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2035	if (pCpu->fFlushAsidBeforeUse)
2036	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2037	}
2038	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2039	* where it is commented out. Support individual entry flushing
2040	* someday. */
2041	#if 0
2042	else
2043	{
2044	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2045	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2046	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2047	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2048
2049	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2050	{
2051	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2052	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2053	{
2054	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2055	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2056	}
2057	else
2058	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2059
2060	pVCpu->hm.s.TlbShootdown.cPages = 0;
2061	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2062	}
2063	else
2064	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2065	}
2066	#endif
2067
2068	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2069	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2070	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2071	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2072	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2073	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2074
2075	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2076	AssertRC(rc);
2077	}
2078
2079
2080	/**
2081	* Flushes the guest TLB entry based on CPU capabilities.
2082	*
2083	* @param pVCpu Pointer to the VMCPU.
2084	* @param pCpu Pointer to the global HM CPU struct.
2085	*/
2086	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2087	{
2088	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2089	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2090	#endif
2091	PVM pVM = pVCpu->CTX_SUFF(pVM);
2092	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2093	{
2094	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2095	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2096	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2097	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2098	default:
2099	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2100	break;
2101	}
2102
2103	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2104	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2105
2106	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2107	}
2108
2109
2110	/**
2111	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2112	* TLB entries from the host TLB before VM-entry.
2113	*
2114	* @returns VBox status code.
2115	* @param pVM Pointer to the VM.
2116	*/
2117	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2118	{
2119	/*
2120	* Determine optimal flush type for Nested Paging.
2121	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2122	* guest execution (see hmR3InitFinalizeR0()).
2123	*/
2124	if (pVM->hm.s.fNestedPaging)
2125	{
2126	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2127	{
2128	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2129	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2130	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2131	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2132	else
2133	{
2134	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2135	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2136	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2137	}
2138
2139	/* Make sure the write-back cacheable memory type for EPT is supported. */
2140	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2141	{
2142	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2143	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2144	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2145	}
2146	}
2147	else
2148	{
2149	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2150	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2151	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2152	}
2153	}
2154
2155	/*
2156	* Determine optimal flush type for VPID.
2157	*/
2158	if (pVM->hm.s.vmx.fVpid)
2159	{
2160	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2161	{
2162	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2164	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2165	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2166	else
2167	{
2168	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2169	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2170	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2171	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2172	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2173	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2174	pVM->hm.s.vmx.fVpid = false;
2175	}
2176	}
2177	else
2178	{
2179	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2180	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2181	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2182	pVM->hm.s.vmx.fVpid = false;
2183	}
2184	}
2185
2186	/*
2187	* Setup the handler for flushing tagged-TLBs.
2188	*/
2189	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2190	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2191	else if (pVM->hm.s.fNestedPaging)
2192	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2193	else if (pVM->hm.s.vmx.fVpid)
2194	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2195	else
2196	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2197	return VINF_SUCCESS;
2198	}
2199
2200
2201	/**
2202	* Sets up pin-based VM-execution controls in the VMCS.
2203	*
2204	* @returns VBox status code.
2205	* @param pVM Pointer to the VM.
2206	* @param pVCpu Pointer to the VMCPU.
2207	*/
2208	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2209	{
2210	AssertPtr(pVM);
2211	AssertPtr(pVCpu);
2212
2213	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2214	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2215
2216	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2217	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2218	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2219
2220	/* Enable the VMX preemption timer. */
2221	if (pVM->hm.s.vmx.fUsePreemptTimer)
2222	{
2223	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2224	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2225	}
2226
2227	if ((val & zap) != val)
2228	{
2229	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2230	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2231	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2232	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2233	}
2234
2235	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2236	AssertRCReturn(rc, rc);
2237
2238	/* Update VCPU with the currently set pin-based VM-execution controls. */
2239	pVCpu->hm.s.vmx.u32PinCtls = val;
2240	return rc;
2241	}
2242
2243
2244	/**
2245	* Sets up processor-based VM-execution controls in the VMCS.
2246	*
2247	* @returns VBox status code.
2248	* @param pVM Pointer to the VM.
2249	* @param pVMCPU Pointer to the VMCPU.
2250	*/
2251	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2252	{
2253	AssertPtr(pVM);
2254	AssertPtr(pVCpu);
2255
2256	int rc = VERR_INTERNAL_ERROR_5;
2257	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2258	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2259
2260	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2261	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2265	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2266	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2267
2268	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2269	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2270	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2271	{
2272	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2273	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2274	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2275	}
2276
2277	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2278	if (!pVM->hm.s.fNestedPaging)
2279	{
2280	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2281	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2282	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2283	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2284	}
2285
2286	/* Use TPR shadowing if supported by the CPU. */
2287	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2288	{
2289	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2290	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2291	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2292	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2293	AssertRCReturn(rc, rc);
2294
2295	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2296	/* CR8 writes causes a VM-exit based on TPR threshold. */
2297	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2298	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2299	}
2300	else
2301	{
2302	/*
2303	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2304	* Set this control only for 64-bit guests.
2305	*/
2306	if (pVM->hm.s.fAllow64BitGuests)
2307	{
2308	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2309	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2310	}
2311	}
2312
2313	/* Use MSR-bitmaps if supported by the CPU. */
2314	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2315	{
2316	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2317
2318	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2319	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2320	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2321	AssertRCReturn(rc, rc);
2322
2323	/*
2324	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2325	* automatically as dedicated fields in the VMCS.
2326	*/
2327	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2331	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2332
2333	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2334	/*
2335	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2336	*/
2337	if ( HMVMX_IS_64BIT_HOST_MODE()
2338	&& pVM->hm.s.fAllow64BitGuests)
2339	{
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2342	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2343	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2344	}
2345	#endif
2346	}
2347
2348	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2349	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2350	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2351
2352	if ((val & zap) != val)
2353	{
2354	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2355	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2356	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2357	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2358	}
2359
2360	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2361	AssertRCReturn(rc, rc);
2362
2363	/* Update VCPU with the currently set processor-based VM-execution controls. */
2364	pVCpu->hm.s.vmx.u32ProcCtls = val;
2365
2366	/*
2367	* Secondary processor-based VM-execution controls.
2368	*/
2369	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2370	{
2371	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2372	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2373
2374	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2375	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2376
2377	if (pVM->hm.s.fNestedPaging)
2378	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2379	else
2380	{
2381	/*
2382	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2383	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2384	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2385	*/
2386	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2387	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2388	}
2389
2390	if (pVM->hm.s.vmx.fVpid)
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2392
2393	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2394	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2395
2396	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2397	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2398	* done dynamically. */
2399	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2400	{
2401	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2402	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2403	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2404	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2405	AssertRCReturn(rc, rc);
2406	}
2407
2408	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2409	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2410
2411	if ((val & zap) != val)
2412	{
2413	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2414	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2415	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2416	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2417	}
2418
2419	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2420	AssertRCReturn(rc, rc);
2421
2422	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2423	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2424	}
2425	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2426	{
2427	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2428	"available\n"));
2429	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2430	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2431	}
2432
2433	return VINF_SUCCESS;
2434	}
2435
2436
2437	/**
2438	* Sets up miscellaneous (everything other than Pin & Processor-based
2439	* VM-execution) control fields in the VMCS.
2440	*
2441	* @returns VBox status code.
2442	* @param pVM Pointer to the VM.
2443	* @param pVCpu Pointer to the VMCPU.
2444	*/
2445	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2446	{
2447	NOREF(pVM);
2448	AssertPtr(pVM);
2449	AssertPtr(pVCpu);
2450
2451	int rc = VERR_GENERAL_FAILURE;
2452
2453	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2454	#if 0
2455	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2456	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2457	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2458
2459	/*
2460	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2461	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2462	* We thus use the exception bitmap to control it rather than use both.
2463	*/
2464	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2465	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2466
2467	/** @todo Explore possibility of using IO-bitmaps. */
2468	/* All IO & IOIO instructions cause VM-exits. */
2469	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2470	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2471
2472	/* Initialize the MSR-bitmap area. */
2473	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2474	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2476	#endif
2477
2478	/* Setup MSR auto-load/store area. */
2479	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2480	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2481	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2482	AssertRCReturn(rc, rc);
2483	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2484	AssertRCReturn(rc, rc);
2485
2486	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2487	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2488	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2489	AssertRCReturn(rc, rc);
2490
2491	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2492	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2493	AssertRCReturn(rc, rc);
2494
2495	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2496	#if 0
2497	/* Setup debug controls */
2498	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2499	AssertRCReturn(rc, rc);
2500	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2501	AssertRCReturn(rc, rc);
2502	#endif
2503
2504	return rc;
2505	}
2506
2507
2508	/**
2509	* Sets up the initial exception bitmap in the VMCS based on static conditions
2510	* (i.e. conditions that cannot ever change after starting the VM).
2511	*
2512	* @returns VBox status code.
2513	* @param pVM Pointer to the VM.
2514	* @param pVCpu Pointer to the VMCPU.
2515	*/
2516	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2517	{
2518	AssertPtr(pVM);
2519	AssertPtr(pVCpu);
2520
2521	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2522
2523	uint32_t u32XcptBitmap = 0;
2524
2525	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2526	if (!pVM->hm.s.fNestedPaging)
2527	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2528
2529	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2530	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2531	AssertRCReturn(rc, rc);
2532	return rc;
2533	}
2534
2535
2536	/**
2537	* Sets up the initial guest-state mask. The guest-state mask is consulted
2538	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2539	* for the nested virtualization case (as it would cause a VM-exit).
2540	*
2541	* @param pVCpu Pointer to the VMCPU.
2542	*/
2543	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2544	{
2545	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2546	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2547	return VINF_SUCCESS;
2548	}
2549
2550
2551	/**
2552	* Does per-VM VT-x initialization.
2553	*
2554	* @returns VBox status code.
2555	* @param pVM Pointer to the VM.
2556	*/
2557	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2558	{
2559	LogFlowFunc(("pVM=%p\n", pVM));
2560
2561	int rc = hmR0VmxStructsAlloc(pVM);
2562	if (RT_FAILURE(rc))
2563	{
2564	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2565	return rc;
2566	}
2567
2568	return VINF_SUCCESS;
2569	}
2570
2571
2572	/**
2573	* Does per-VM VT-x termination.
2574	*
2575	* @returns VBox status code.
2576	* @param pVM Pointer to the VM.
2577	*/
2578	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2579	{
2580	LogFlowFunc(("pVM=%p\n", pVM));
2581
2582	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2583	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2584	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2585	#endif
2586	hmR0VmxStructsFree(pVM);
2587	return VINF_SUCCESS;
2588	}
2589
2590
2591	/**
2592	* Sets up the VM for execution under VT-x.
2593	* This function is only called once per-VM during initialization.
2594	*
2595	* @returns VBox status code.
2596	* @param pVM Pointer to the VM.
2597	*/
2598	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2599	{
2600	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2601	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2602
2603	LogFlowFunc(("pVM=%p\n", pVM));
2604
2605	/*
2606	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2607	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2608	*/
2609	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2610	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2611	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2612	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2613	{
2614	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2615	return VERR_INTERNAL_ERROR;
2616	}
2617
2618	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2619	/*
2620	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2621	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2622	*/
2623	if ( pVM->hm.s.fAllow64BitGuests
2624	&& !HMVMX_IS_64BIT_HOST_MODE())
2625	{
2626	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2627	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2628	}
2629	#endif
2630
2631	/* Initialize these always, see hmR3InitFinalizeR0().*/
2632	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2633	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2634
2635	/* Setup the tagged-TLB flush handlers. */
2636	int rc = hmR0VmxSetupTaggedTlb(pVM);
2637	if (RT_FAILURE(rc))
2638	{
2639	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2640	return rc;
2641	}
2642
2643	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2644	{
2645	PVMCPU pVCpu = &pVM->aCpus[i];
2646	AssertPtr(pVCpu);
2647	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2648
2649	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2650	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2651
2652	/* Set revision dword at the beginning of the VMCS structure. */
2653	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2654
2655	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2656	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2657	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2658	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2659
2660	/* Load this VMCS as the current VMCS. */
2661	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2662	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2663	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2664
2665	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2666	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2667	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2668
2669	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2670	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2671	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2672
2673	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2674	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2675	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2676
2677	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2678	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2679	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2680
2681	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2682	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2683	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2684
2685	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2686	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2687	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2688	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2689	#endif
2690
2691	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2692	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2693	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2694	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2695
2696	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2697
2698	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2699	}
2700
2701	return VINF_SUCCESS;
2702	}
2703
2704
2705	/**
2706	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2707	* the VMCS.
2708	*
2709	* @returns VBox status code.
2710	* @param pVM Pointer to the VM.
2711	* @param pVCpu Pointer to the VMCPU.
2712	*/
2713	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2714	{
2715	NOREF(pVM); NOREF(pVCpu);
2716
2717	RTCCUINTREG uReg = ASMGetCR0();
2718	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2719	AssertRCReturn(rc, rc);
2720
2721	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2722	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2723	if (HMVMX_IS_64BIT_HOST_MODE())
2724	{
2725	uint64_t uRegCR3 = HMR0Get64bitCR3();
2726	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2727	}
2728	else
2729	#endif
2730	{
2731	uReg = ASMGetCR3();
2732	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2733	}
2734	AssertRCReturn(rc, rc);
2735
2736	uReg = ASMGetCR4();
2737	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2738	AssertRCReturn(rc, rc);
2739	return rc;
2740	}
2741
2742
2743	#if HC_ARCH_BITS == 64
2744	/**
2745	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2746	* requirements. See hmR0VmxSaveHostSegmentRegs().
2747	*/
2748	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2749	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2750	{ \
2751	bool fValidSelector = true; \
2752	if ((selValue) & X86_SEL_LDT) \
2753	{ \
2754	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2755	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2756	} \
2757	if (fValidSelector) \
2758	{ \
2759	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2760	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2761	} \
2762	(selValue) = 0; \
2763	}
2764	#endif
2765
2766
2767	/**
2768	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2769	* the host-state area in the VMCS.
2770	*
2771	* @returns VBox status code.
2772	* @param pVM Pointer to the VM.
2773	* @param pVCpu Pointer to the VMCPU.
2774	*/
2775	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2776	{
2777	NOREF(pVM);
2778	int rc = VERR_INTERNAL_ERROR_5;
2779
2780	/*
2781	* Host DS, ES, FS and GS segment registers.
2782	*/
2783	#if HC_ARCH_BITS == 64
2784	RTSEL uSelDS = ASMGetDS();
2785	RTSEL uSelES = ASMGetES();
2786	RTSEL uSelFS = ASMGetFS();
2787	RTSEL uSelGS = ASMGetGS();
2788	#else
2789	RTSEL uSelDS = 0;
2790	RTSEL uSelES = 0;
2791	RTSEL uSelFS = 0;
2792	RTSEL uSelGS = 0;
2793	#endif
2794
2795	/* Recalculate which host-state bits need to be manually restored. */
2796	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2797
2798	/*
2799	* Host CS and SS segment registers.
2800	*/
2801	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2802	RTSEL uSelCS;
2803	RTSEL uSelSS;
2804	if (HMVMX_IS_64BIT_HOST_MODE())
2805	{
2806	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2807	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2808	}
2809	else
2810	{
2811	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2812	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2813	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2814	}
2815	#else
2816	RTSEL uSelCS = ASMGetCS();
2817	RTSEL uSelSS = ASMGetSS();
2818	#endif
2819
2820	/*
2821	* Host TR segment register.
2822	*/
2823	RTSEL uSelTR = ASMGetTR();
2824
2825	#if HC_ARCH_BITS == 64
2826	/*
2827	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2828	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2829	*/
2830	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2831	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2832	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2833	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2834	# undef VMXLOCAL_ADJUST_HOST_SEG
2835	#endif
2836
2837	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2838	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2839	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2840	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2841	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2842	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2843	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2844	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2845	Assert(uSelCS);
2846	Assert(uSelTR);
2847
2848	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2849	#if 0
2850	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2851	Assert(uSelSS != 0);
2852	#endif
2853
2854	/* Write these host selector fields into the host-state area in the VMCS. */
2855	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2856	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2857	#if HC_ARCH_BITS == 64
2858	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2859	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2860	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2861	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2862	#endif
2863	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2864
2865	/*
2866	* Host GDTR and IDTR.
2867	*/
2868	RTGDTR Gdtr;
2869	RT_ZERO(Gdtr);
2870	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2871	if (HMVMX_IS_64BIT_HOST_MODE())
2872	{
2873	X86XDTR64 Gdtr64;
2874	X86XDTR64 Idtr64;
2875	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2876	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2877	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2878
2879	Gdtr.cbGdt = Gdtr64.cb;
2880	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2881	}
2882	else
2883	#endif
2884	{
2885	RTIDTR Idtr;
2886	ASMGetGDTR(&Gdtr);
2887	ASMGetIDTR(&Idtr);
2888	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2889	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2890
2891	#if HC_ARCH_BITS == 64
2892	/*
2893	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2894	* maximum limit (0xffff) on every VM-exit.
2895	*/
2896	if (Gdtr.cbGdt != 0xffff)
2897	{
2898	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2899	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2900	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2901	}
2902
2903	/*
2904	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2905	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2906	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2907	*/
2908	if (Idtr.cbIdt < 0x0fff)
2909	{
2910	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2911	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2912	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2913	}
2914	#endif
2915	}
2916
2917	/*
2918	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2919	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2920	*/
2921	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2922	{
2923	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2924	return VERR_VMX_INVALID_HOST_STATE;
2925	}
2926
2927	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2928	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2929	if (HMVMX_IS_64BIT_HOST_MODE())
2930	{
2931	/* We need the 64-bit TR base for hybrid darwin. */
2932	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2933	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2934	}
2935	else
2936	#endif
2937	{
2938	uintptr_t uTRBase;
2939	#if HC_ARCH_BITS == 64
2940	uTRBase = X86DESC64_BASE(pDesc);
2941
2942	/*
2943	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2944	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2945	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2946	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2947	*
2948	* [1] See Intel spec. 3.5 "System Descriptor Types".
2949	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2950	*/
2951	Assert(pDesc->System.u4Type == 11);
2952	if ( pDesc->System.u16LimitLow != 0x67
2953	\|\| pDesc->System.u4LimitHigh)
2954	{
2955	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2956	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2957
2958	/* Store the GDTR here as we need it while restoring TR. */
2959	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2960	}
2961	#else
2962	uTRBase = X86DESC_BASE(pDesc);
2963	#endif
2964	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2965	}
2966	AssertRCReturn(rc, rc);
2967
2968	/*
2969	* Host FS base and GS base.
2970	*/
2971	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2972	if (HMVMX_IS_64BIT_HOST_MODE())
2973	{
2974	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2975	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2976	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2977	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2978
2979	# if HC_ARCH_BITS == 64
2980	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2981	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2982	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2983	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2984	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2985	# endif
2986	}
2987	#endif
2988	return rc;
2989	}
2990
2991
2992	/**
2993	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2994	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2995	* the host after every successful VM-exit.
2996	*
2997	* @returns VBox status code.
2998	* @param pVM Pointer to the VM.
2999	* @param pVCpu Pointer to the VMCPU.
3000	*
3001	* @remarks No-long-jump zone!!!
3002	*/
3003	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3004	{
3005	NOREF(pVM);
3006
3007	AssertPtr(pVCpu);
3008	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3009
3010	int rc = VINF_SUCCESS;
3011	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3012	if ( HMVMX_IS_64BIT_HOST_MODE()
3013	&& pVM->hm.s.fAllow64BitGuests)
3014	{
3015	hmR0VmxLazySaveHostMsrs(pVCpu);
3016	}
3017	#endif
3018
3019	if (pVCpu->hm.s.vmx.cMsrs > 0)
3020	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3021
3022	/*
3023	* Host Sysenter MSRs.
3024	*/
3025	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3026	AssertRCReturn(rc, rc);
3027	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3028	if (HMVMX_IS_64BIT_HOST_MODE())
3029	{
3030	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3031	AssertRCReturn(rc, rc);
3032	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3033	}
3034	else
3035	{
3036	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3037	AssertRCReturn(rc, rc);
3038	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3039	}
3040	#elif HC_ARCH_BITS == 32
3041	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3042	AssertRCReturn(rc, rc);
3043	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3044	#else
3045	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3046	AssertRCReturn(rc, rc);
3047	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3048	#endif
3049	AssertRCReturn(rc, rc);
3050
3051	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3052	* hmR0VmxSetupExitCtls() !! */
3053	return rc;
3054	}
3055
3056
3057	/**
3058	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3059	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3060	* controls".
3061	*
3062	* @returns VBox status code.
3063	* @param pVCpu Pointer to the VMCPU.
3064	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3065	* out-of-sync. Make sure to update the required fields
3066	* before using them.
3067	*
3068	* @remarks No-long-jump zone!!!
3069	*/
3070	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3071	{
3072	int rc = VINF_SUCCESS;
3073	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3074	{
3075	PVM pVM = pVCpu->CTX_SUFF(pVM);
3076	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3077	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3078
3079	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3080	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3081
3082	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3083	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3084	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3085	else
3086	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3087
3088	/*
3089	* The following should -not- be set (since we're not in SMM mode):
3090	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3091	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3092	*/
3093
3094	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3095	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3096	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3097
3098	if ((val & zap) != val)
3099	{
3100	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3101	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3102	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3103	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3104	}
3105
3106	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3107	AssertRCReturn(rc, rc);
3108
3109	/* Update VCPU with the currently set VM-exit controls. */
3110	pVCpu->hm.s.vmx.u32EntryCtls = val;
3111	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3112	}
3113	return rc;
3114	}
3115
3116
3117	/**
3118	* Sets up the VM-exit controls in the VMCS.
3119	*
3120	* @returns VBox status code.
3121	* @param pVM Pointer to the VM.
3122	* @param pVCpu Pointer to the VMCPU.
3123	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3124	* out-of-sync. Make sure to update the required fields
3125	* before using them.
3126	*
3127	* @remarks requires EFER.
3128	*/
3129	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3130	{
3131	NOREF(pMixedCtx);
3132
3133	int rc = VINF_SUCCESS;
3134	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3135	{
3136	PVM pVM = pVCpu->CTX_SUFF(pVM);
3137	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3138	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3139
3140	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3141	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3142
3143	/*
3144	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3145	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3146	*/
3147	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3148	if (HMVMX_IS_64BIT_HOST_MODE())
3149	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3150	else
3151	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3152	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3153	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3154	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3155	else
3156	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3157	#endif
3158
3159	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3160	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3161
3162	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3163	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3164	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3165	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3166	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3167
3168	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3169	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3170
3171	if ((val & zap) != val)
3172	{
3173	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3174	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3175	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3176	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3177	}
3178
3179	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3180	AssertRCReturn(rc, rc);
3181
3182	/* Update VCPU with the currently set VM-exit controls. */
3183	pVCpu->hm.s.vmx.u32ExitCtls = val;
3184	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3185	}
3186	return rc;
3187	}
3188
3189
3190	/**
3191	* Loads the guest APIC and related state.
3192	*
3193	* @returns VBox status code.
3194	* @param pVM Pointer to the VM.
3195	* @param pVCpu Pointer to the VMCPU.
3196	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3197	* out-of-sync. Make sure to update the required fields
3198	* before using them.
3199	*/
3200	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3201	{
3202	NOREF(pMixedCtx);
3203
3204	int rc = VINF_SUCCESS;
3205	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3206	{
3207	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3208	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3209	{
3210	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3211
3212	bool fPendingIntr = false;
3213	uint8_t u8Tpr = 0;
3214	uint8_t u8PendingIntr = 0;
3215	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3216	AssertRCReturn(rc, rc);
3217
3218	/*
3219	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3220	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3221	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3222	* the interrupt when we VM-exit for other reasons.
3223	*/
3224	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3225	uint32_t u32TprThreshold = 0;
3226	if (fPendingIntr)
3227	{
3228	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3229	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3230	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3231	if (u8PendingPriority <= u8TprPriority)
3232	u32TprThreshold = u8PendingPriority;
3233	else
3234	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3235	}
3236	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3237
3238	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3239	AssertRCReturn(rc, rc);
3240	}
3241
3242	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3243	}
3244	return rc;
3245	}
3246
3247
3248	/**
3249	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3250	*
3251	* @returns Guest's interruptibility-state.
3252	* @param pVCpu Pointer to the VMCPU.
3253	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3254	* out-of-sync. Make sure to update the required fields
3255	* before using them.
3256	*
3257	* @remarks No-long-jump zone!!!
3258	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3259	*/
3260	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3261	{
3262	/*
3263	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3264	* inhibit interrupts or clear any existing interrupt-inhibition.
3265	*/
3266	uint32_t uIntrState = 0;
3267	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3268	{
3269	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3270	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
3271	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3272	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3273	{
3274	/*
3275	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3276	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3277	*/
3278	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3279	}
3280	else if (pMixedCtx->eflags.Bits.u1IF)
3281	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3282	else
3283	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3284	}
3285	return uIntrState;
3286	}
3287
3288
3289	/**
3290	* Loads the guest's interruptibility-state into the guest-state area in the
3291	* VMCS.
3292	*
3293	* @returns VBox status code.
3294	* @param pVCpu Pointer to the VMCPU.
3295	* @param uIntrState The interruptibility-state to set.
3296	*/
3297	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3298	{
3299	NOREF(pVCpu);
3300	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3301	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3302	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3303	AssertRCReturn(rc, rc);
3304	return rc;
3305	}
3306
3307
3308	/**
3309	* Loads the guest's RIP into the guest-state area in the VMCS.
3310	*
3311	* @returns VBox status code.
3312	* @param pVCpu Pointer to the VMCPU.
3313	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3314	* out-of-sync. Make sure to update the required fields
3315	* before using them.
3316	*
3317	* @remarks No-long-jump zone!!!
3318	*/
3319	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3320	{
3321	int rc = VINF_SUCCESS;
3322	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3323	{
3324	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3325	AssertRCReturn(rc, rc);
3326
3327	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3328	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3329	}
3330	return rc;
3331	}
3332
3333
3334	/**
3335	* Loads the guest's RSP into the guest-state area in the VMCS.
3336	*
3337	* @returns VBox status code.
3338	* @param pVCpu Pointer to the VMCPU.
3339	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3340	* out-of-sync. Make sure to update the required fields
3341	* before using them.
3342	*
3343	* @remarks No-long-jump zone!!!
3344	*/
3345	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3346	{
3347	int rc = VINF_SUCCESS;
3348	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3349	{
3350	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3351	AssertRCReturn(rc, rc);
3352
3353	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3354	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3355	}
3356	return rc;
3357	}
3358
3359
3360	/**
3361	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3362	*
3363	* @returns VBox status code.
3364	* @param pVCpu Pointer to the VMCPU.
3365	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3366	* out-of-sync. Make sure to update the required fields
3367	* before using them.
3368	*
3369	* @remarks No-long-jump zone!!!
3370	*/
3371	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3372	{
3373	int rc = VINF_SUCCESS;
3374	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3375	{
3376	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3377	Let us assert it as such and use 32-bit VMWRITE. */
3378	Assert(!(pMixedCtx->rflags.u64 >> 32));
3379	X86EFLAGS Eflags = pMixedCtx->eflags;
3380	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3381	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3382
3383	/*
3384	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3385	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3386	*/
3387	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3388	{
3389	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3390	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3391	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3392	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3393	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3394	}
3395
3396	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3397	AssertRCReturn(rc, rc);
3398
3399	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3400	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3401	}
3402	return rc;
3403	}
3404
3405
3406	/**
3407	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3408	*
3409	* @returns VBox status code.
3410	* @param pVCpu Pointer to the VMCPU.
3411	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3412	* out-of-sync. Make sure to update the required fields
3413	* before using them.
3414	*
3415	* @remarks No-long-jump zone!!!
3416	*/
3417	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3418	{
3419	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3420	AssertRCReturn(rc, rc);
3421	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3422	AssertRCReturn(rc, rc);
3423	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3424	AssertRCReturn(rc, rc);
3425	return rc;
3426	}
3427
3428
3429	/**
3430	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3431	* CR0 is partially shared with the host and we have to consider the FPU bits.
3432	*
3433	* @returns VBox status code.
3434	* @param pVM Pointer to the VM.
3435	* @param pVCpu Pointer to the VMCPU.
3436	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3437	* out-of-sync. Make sure to update the required fields
3438	* before using them.
3439	*
3440	* @remarks No-long-jump zone!!!
3441	*/
3442	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3443	{
3444	/*
3445	* Guest CR0.
3446	* Guest FPU.
3447	*/
3448	int rc = VINF_SUCCESS;
3449	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3450	{
3451	Assert(!(pMixedCtx->cr0 >> 32));
3452	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3453	PVM pVM = pVCpu->CTX_SUFF(pVM);
3454
3455	/* The guest's view (read access) of its CR0 is unblemished. */
3456	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3457	AssertRCReturn(rc, rc);
3458	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3459
3460	/* Setup VT-x's view of the guest CR0. */
3461	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3462	if (pVM->hm.s.fNestedPaging)
3463	{
3464	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3465	{
3466	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3467	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3468	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3469	}
3470	else
3471	{
3472	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3473	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3474	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3475	}
3476
3477	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3478	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3479	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3480
3481	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3482	AssertRCReturn(rc, rc);
3483	}
3484	else
3485	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3486
3487	/*
3488	* Guest FPU bits.
3489	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3490	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3491	*/
3492	u32GuestCR0 \|= X86_CR0_NE;
3493	bool fInterceptNM = false;
3494	if (CPUMIsGuestFPUStateActive(pVCpu))
3495	{
3496	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3497	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3498	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3499	}
3500	else
3501	{
3502	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3503	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3504	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3505	}
3506
3507	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3508	bool fInterceptMF = false;
3509	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3510	fInterceptMF = true;
3511
3512	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3513	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3514	{
3515	Assert(PDMVmmDevHeapIsEnabled(pVM));
3516	Assert(pVM->hm.s.vmx.pRealModeTSS);
3517	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3518	fInterceptNM = true;
3519	fInterceptMF = true;
3520	}
3521	else
3522	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3523
3524	if (fInterceptNM)
3525	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3526	else
3527	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3528
3529	if (fInterceptMF)
3530	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3531	else
3532	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3533
3534	/* Additional intercepts for debugging, define these yourself explicitly. */
3535	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3536	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3537	\| RT_BIT(X86_XCPT_BP)
3538	\| RT_BIT(X86_XCPT_DB)
3539	\| RT_BIT(X86_XCPT_DE)
3540	\| RT_BIT(X86_XCPT_NM)
3541	\| RT_BIT(X86_XCPT_UD)
3542	\| RT_BIT(X86_XCPT_NP)
3543	\| RT_BIT(X86_XCPT_SS)
3544	\| RT_BIT(X86_XCPT_GP)
3545	\| RT_BIT(X86_XCPT_PF)
3546	\| RT_BIT(X86_XCPT_MF)
3547	;
3548	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3549	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3550	#endif
3551
3552	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3553
3554	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3555	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3556	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3557	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3558	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3559	else
3560	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3561
3562	u32GuestCR0 \|= uSetCR0;
3563	u32GuestCR0 &= uZapCR0;
3564	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3565
3566	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3567	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3568	AssertRCReturn(rc, rc);
3569	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3570	AssertRCReturn(rc, rc);
3571	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3572
3573	/*
3574	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3575	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3576	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3577	*/
3578	uint32_t u32CR0Mask = 0;
3579	u32CR0Mask = X86_CR0_PE
3580	\| X86_CR0_NE
3581	\| X86_CR0_WP
3582	\| X86_CR0_PG
3583	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3584	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3585	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3586
3587	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3588	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3589	* and @bugref{6944}. */
3590	#if 0
3591	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3592	u32CR0Mask &= ~X86_CR0_PE;
3593	#endif
3594	if (pVM->hm.s.fNestedPaging)
3595	u32CR0Mask &= ~X86_CR0_WP;
3596
3597	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3598	if (fInterceptNM)
3599	{
3600	u32CR0Mask \|= X86_CR0_TS
3601	\| X86_CR0_MP;
3602	}
3603
3604	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3605	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3606	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3607	AssertRCReturn(rc, rc);
3608	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3609
3610	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3611	}
3612	return rc;
3613	}
3614
3615
3616	/**
3617	* Loads the guest control registers (CR3, CR4) into the guest-state area
3618	* in the VMCS.
3619	*
3620	* @returns VBox status code.
3621	* @param pVM Pointer to the VM.
3622	* @param pVCpu Pointer to the VMCPU.
3623	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3624	* out-of-sync. Make sure to update the required fields
3625	* before using them.
3626	*
3627	* @remarks No-long-jump zone!!!
3628	*/
3629	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3630	{
3631	int rc = VINF_SUCCESS;
3632	PVM pVM = pVCpu->CTX_SUFF(pVM);
3633
3634	/*
3635	* Guest CR2.
3636	* It's always loaded in the assembler code. Nothing to do here.
3637	*/
3638
3639	/*
3640	* Guest CR3.
3641	*/
3642	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3643	{
3644	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3645	if (pVM->hm.s.fNestedPaging)
3646	{
3647	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3648
3649	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3650	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3651	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3652	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3653
3654	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3655	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3656	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3657
3658	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3659	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3660	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3661	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3662
3663	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3664	AssertRCReturn(rc, rc);
3665	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3666
3667	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3668	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3669	{
3670	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3671	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3672	{
3673	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3674	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3675	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3676	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3677	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3678	}
3679
3680	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3681	have Unrestricted Execution to handle the guest when it's not using paging. */
3682	GCPhysGuestCR3 = pMixedCtx->cr3;
3683	}
3684	else
3685	{
3686	/*
3687	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3688	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3689	* EPT takes care of translating it to host-physical addresses.
3690	*/
3691	RTGCPHYS GCPhys;
3692	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3693	Assert(PDMVmmDevHeapIsEnabled(pVM));
3694
3695	/* We obtain it here every time as the guest could have relocated this PCI region. */
3696	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3697	AssertRCReturn(rc, rc);
3698
3699	GCPhysGuestCR3 = GCPhys;
3700	}
3701
3702	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3703	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3704	}
3705	else
3706	{
3707	/* Non-nested paging case, just use the hypervisor's CR3. */
3708	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3709
3710	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3711	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3712	}
3713	AssertRCReturn(rc, rc);
3714
3715	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3716	}
3717
3718	/*
3719	* Guest CR4.
3720	*/
3721	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3722	{
3723	Assert(!(pMixedCtx->cr4 >> 32));
3724	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3725
3726	/* The guest's view of its CR4 is unblemished. */
3727	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3728	AssertRCReturn(rc, rc);
3729	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3730
3731	/* Setup VT-x's view of the guest CR4. */
3732	/*
3733	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3734	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3735	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3736	*/
3737	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3738	{
3739	Assert(pVM->hm.s.vmx.pRealModeTSS);
3740	Assert(PDMVmmDevHeapIsEnabled(pVM));
3741	u32GuestCR4 &= ~X86_CR4_VME;
3742	}
3743
3744	if (pVM->hm.s.fNestedPaging)
3745	{
3746	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3747	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3748	{
3749	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3750	u32GuestCR4 \|= X86_CR4_PSE;
3751	/* Our identity mapping is a 32-bit page directory. */
3752	u32GuestCR4 &= ~X86_CR4_PAE;
3753	}
3754	/* else use guest CR4.*/
3755	}
3756	else
3757	{
3758	/*
3759	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3760	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3761	*/
3762	switch (pVCpu->hm.s.enmShadowMode)
3763	{
3764	case PGMMODE_REAL: /* Real-mode. */
3765	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3766	case PGMMODE_32_BIT: /* 32-bit paging. */
3767	{
3768	u32GuestCR4 &= ~X86_CR4_PAE;
3769	break;
3770	}
3771
3772	case PGMMODE_PAE: /* PAE paging. */
3773	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3774	{
3775	u32GuestCR4 \|= X86_CR4_PAE;
3776	break;
3777	}
3778
3779	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3780	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3781	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3782	break;
3783	#endif
3784	default:
3785	AssertFailed();
3786	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3787	}
3788	}
3789
3790	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3791	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3792	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3793	u32GuestCR4 \|= uSetCR4;
3794	u32GuestCR4 &= uZapCR4;
3795
3796	/* Write VT-x's view of the guest CR4 into the VMCS. */
3797	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3798	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3799	AssertRCReturn(rc, rc);
3800
3801	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3802	uint32_t u32CR4Mask = 0;
3803	u32CR4Mask = X86_CR4_VME
3804	\| X86_CR4_PAE
3805	\| X86_CR4_PGE
3806	\| X86_CR4_PSE
3807	\| X86_CR4_VMXE;
3808	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3809	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3810	AssertRCReturn(rc, rc);
3811
3812	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3813	}
3814	return rc;
3815	}
3816
3817
3818	/**
3819	* Loads the guest debug registers into the guest-state area in the VMCS.
3820	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3821	*
3822	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3823	*
3824	* @returns VBox status code.
3825	* @param pVCpu Pointer to the VMCPU.
3826	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3827	* out-of-sync. Make sure to update the required fields
3828	* before using them.
3829	*
3830	* @remarks No-long-jump zone!!!
3831	*/
3832	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3833	{
3834	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3835	return VINF_SUCCESS;
3836
3837	#ifdef VBOX_STRICT
3838	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3839	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3840	{
3841	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3842	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3843	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3844	}
3845	#endif
3846
3847	int rc;
3848	PVM pVM = pVCpu->CTX_SUFF(pVM);
3849	bool fInterceptDB = false;
3850	bool fInterceptMovDRx = false;
3851	if ( pVCpu->hm.s.fSingleInstruction
3852	\|\| DBGFIsStepping(pVCpu))
3853	{
3854	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3855	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3856	{
3857	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3858	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3859	AssertRCReturn(rc, rc);
3860	Assert(fInterceptDB == false);
3861	}
3862	else
3863	{
3864	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3865	pVCpu->hm.s.fClearTrapFlag = true;
3866	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3867	fInterceptDB = true;
3868	}
3869	}
3870
3871	if ( fInterceptDB
3872	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3873	{
3874	/*
3875	* Use the combined guest and host DRx values found in the hypervisor
3876	* register set because the debugger has breakpoints active or someone
3877	* is single stepping on the host side without a monitor trap flag.
3878	*
3879	* Note! DBGF expects a clean DR6 state before executing guest code.
3880	*/
3881	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3882	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3883	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3884	{
3885	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3886	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3887	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3888	}
3889	else
3890	#endif
3891	if (!CPUMIsHyperDebugStateActive(pVCpu))
3892	{
3893	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3894	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3895	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3896	}
3897
3898	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3899	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3900	AssertRCReturn(rc, rc);
3901
3902	pVCpu->hm.s.fUsingHyperDR7 = true;
3903	fInterceptDB = true;
3904	fInterceptMovDRx = true;
3905	}
3906	else
3907	{
3908	/*
3909	* If the guest has enabled debug registers, we need to load them prior to
3910	* executing guest code so they'll trigger at the right time.
3911	*/
3912	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3913	{
3914	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3915	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3916	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3917	{
3918	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3919	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3920	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3921	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3922	}
3923	else
3924	#endif
3925	if (!CPUMIsGuestDebugStateActive(pVCpu))
3926	{
3927	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3928	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3929	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3930	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3931	}
3932	Assert(!fInterceptDB);
3933	Assert(!fInterceptMovDRx);
3934	}
3935	/*
3936	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3937	* must intercept #DB in order to maintain a correct DR6 guest value.
3938	*/
3939	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3940	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3941	&& !CPUMIsGuestDebugStateActive(pVCpu))
3942	#else
3943	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3944	#endif
3945	{
3946	fInterceptMovDRx = true;
3947	fInterceptDB = true;
3948	}
3949
3950	/* Update guest DR7. */
3951	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3952	AssertRCReturn(rc, rc);
3953
3954	pVCpu->hm.s.fUsingHyperDR7 = false;
3955	}
3956
3957	/*
3958	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3959	*/
3960	if (fInterceptDB)
3961	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3962	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3963	{
3964	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3965	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3966	#endif
3967	}
3968	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3969	AssertRCReturn(rc, rc);
3970
3971	/*
3972	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3973	*/
3974	if (fInterceptMovDRx)
3975	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3976	else
3977	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3978	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3979	AssertRCReturn(rc, rc);
3980
3981	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3982	return VINF_SUCCESS;
3983	}
3984
3985
3986	#ifdef VBOX_STRICT
3987	/**
3988	* Strict function to validate segment registers.
3989	*
3990	* @remarks ASSUMES CR0 is up to date.
3991	*/
3992	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3993	{
3994	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3995	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3996	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3997	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3998	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3999	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4000	{
4001	/* Protected mode checks */
4002	/* CS */
4003	Assert(pCtx->cs.Attr.n.u1Present);
4004	Assert(!(pCtx->cs.Attr.u & 0xf00));
4005	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4006	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4007	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4008	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4009	\|\| (pCtx->cs.Attr.n.u1Granularity));
4010	/* CS cannot be loaded with NULL in protected mode. */
4011	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4012	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4013	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4014	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4015	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4016	else
4017	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4018	/* SS */
4019	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4020	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4021	if ( !(pCtx->cr0 & X86_CR0_PE)
4022	\|\| pCtx->cs.Attr.n.u4Type == 3)
4023	{
4024	Assert(!pCtx->ss.Attr.n.u2Dpl);
4025	}
4026	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4027	{
4028	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4029	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4030	Assert(pCtx->ss.Attr.n.u1Present);
4031	Assert(!(pCtx->ss.Attr.u & 0xf00));
4032	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4033	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4034	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4035	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4036	\|\| (pCtx->ss.Attr.n.u1Granularity));
4037	}
4038	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4039	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4040	{
4041	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4042	Assert(pCtx->ds.Attr.n.u1Present);
4043	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4044	Assert(!(pCtx->ds.Attr.u & 0xf00));
4045	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4046	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4047	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4048	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4049	\|\| (pCtx->ds.Attr.n.u1Granularity));
4050	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4051	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4052	}
4053	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4054	{
4055	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4056	Assert(pCtx->es.Attr.n.u1Present);
4057	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4058	Assert(!(pCtx->es.Attr.u & 0xf00));
4059	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4060	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4061	\|\| !(pCtx->es.Attr.n.u1Granularity));
4062	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4063	\|\| (pCtx->es.Attr.n.u1Granularity));
4064	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4065	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4066	}
4067	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4068	{
4069	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4070	Assert(pCtx->fs.Attr.n.u1Present);
4071	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4072	Assert(!(pCtx->fs.Attr.u & 0xf00));
4073	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4074	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4075	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4076	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4077	\|\| (pCtx->fs.Attr.n.u1Granularity));
4078	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4079	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4080	}
4081	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4082	{
4083	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4084	Assert(pCtx->gs.Attr.n.u1Present);
4085	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4086	Assert(!(pCtx->gs.Attr.u & 0xf00));
4087	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4088	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4089	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4090	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4091	\|\| (pCtx->gs.Attr.n.u1Granularity));
4092	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4093	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4094	}
4095	/* 64-bit capable CPUs. */
4096	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4097	Assert(!(pCtx->cs.u64Base >> 32));
4098	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4099	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4100	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4101	# endif
4102	}
4103	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4104	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4105	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4106	{
4107	/* Real and v86 mode checks. */
4108	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4109	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4110	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4111	{
4112	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4113	}
4114	else
4115	{
4116	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4117	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4118	}
4119
4120	/* CS */
4121	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4122	Assert(pCtx->cs.u32Limit == 0xffff);
4123	Assert(u32CSAttr == 0xf3);
4124	/* SS */
4125	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4126	Assert(pCtx->ss.u32Limit == 0xffff);
4127	Assert(u32SSAttr == 0xf3);
4128	/* DS */
4129	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4130	Assert(pCtx->ds.u32Limit == 0xffff);
4131	Assert(u32DSAttr == 0xf3);
4132	/* ES */
4133	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4134	Assert(pCtx->es.u32Limit == 0xffff);
4135	Assert(u32ESAttr == 0xf3);
4136	/* FS */
4137	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4138	Assert(pCtx->fs.u32Limit == 0xffff);
4139	Assert(u32FSAttr == 0xf3);
4140	/* GS */
4141	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4142	Assert(pCtx->gs.u32Limit == 0xffff);
4143	Assert(u32GSAttr == 0xf3);
4144	/* 64-bit capable CPUs. */
4145	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4146	Assert(!(pCtx->cs.u64Base >> 32));
4147	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4148	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4149	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4150	# endif
4151	}
4152	}
4153	#endif /* VBOX_STRICT */
4154
4155
4156	/**
4157	* Writes a guest segment register into the guest-state area in the VMCS.
4158	*
4159	* @returns VBox status code.
4160	* @param pVCpu Pointer to the VMCPU.
4161	* @param idxSel Index of the selector in the VMCS.
4162	* @param idxLimit Index of the segment limit in the VMCS.
4163	* @param idxBase Index of the segment base in the VMCS.
4164	* @param idxAccess Index of the access rights of the segment in the VMCS.
4165	* @param pSelReg Pointer to the segment selector.
4166	*
4167	* @remarks No-long-jump zone!!!
4168	*/
4169	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4170	uint32_t idxAccess, PCPUMSELREG pSelReg)
4171	{
4172	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4173	AssertRCReturn(rc, rc);
4174	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4175	AssertRCReturn(rc, rc);
4176	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4177	AssertRCReturn(rc, rc);
4178
4179	uint32_t u32Access = pSelReg->Attr.u;
4180	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4181	{
4182	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4183	u32Access = 0xf3;
4184	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4185	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4186	}
4187	else
4188	{
4189	/*
4190	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4191	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4192	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4193	* loaded in protected-mode have their attribute as 0.
4194	*/
4195	if (!u32Access)
4196	u32Access = X86DESCATTR_UNUSABLE;
4197	}
4198
4199	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4200	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4201	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4202
4203	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4204	AssertRCReturn(rc, rc);
4205	return rc;
4206	}
4207
4208
4209	/**
4210	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4211	* into the guest-state area in the VMCS.
4212	*
4213	* @returns VBox status code.
4214	* @param pVM Pointer to the VM.
4215	* @param pVCPU Pointer to the VMCPU.
4216	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4217	* out-of-sync. Make sure to update the required fields
4218	* before using them.
4219	*
4220	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4221	* @remarks No-long-jump zone!!!
4222	*/
4223	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4224	{
4225	int rc = VERR_INTERNAL_ERROR_5;
4226	PVM pVM = pVCpu->CTX_SUFF(pVM);
4227
4228	/*
4229	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4230	*/
4231	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4232	{
4233	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4234	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4235	{
4236	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4237	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4238	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4239	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4240	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4241	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4242	}
4243
4244	#ifdef VBOX_WITH_REM
4245	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4246	{
4247	Assert(pVM->hm.s.vmx.pRealModeTSS);
4248	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4249	if ( pVCpu->hm.s.vmx.fWasInRealMode
4250	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4251	{
4252	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4253	in real-mode (e.g. OpenBSD 4.0) */
4254	REMFlushTBs(pVM);
4255	Log4(("Load: Switch to protected mode detected!\n"));
4256	pVCpu->hm.s.vmx.fWasInRealMode = false;
4257	}
4258	}
4259	#endif
4260	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4261	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4262	AssertRCReturn(rc, rc);
4263	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4264	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4265	AssertRCReturn(rc, rc);
4266	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4267	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4268	AssertRCReturn(rc, rc);
4269	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4270	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4271	AssertRCReturn(rc, rc);
4272	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4273	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4274	AssertRCReturn(rc, rc);
4275	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4276	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4277	AssertRCReturn(rc, rc);
4278
4279	#ifdef VBOX_STRICT
4280	/* Validate. */
4281	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4282	#endif
4283
4284	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4285	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4286	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4287	}
4288
4289	/*
4290	* Guest TR.
4291	*/
4292	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4293	{
4294	/*
4295	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4296	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4297	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4298	*/
4299	uint16_t u16Sel = 0;
4300	uint32_t u32Limit = 0;
4301	uint64_t u64Base = 0;
4302	uint32_t u32AccessRights = 0;
4303
4304	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4305	{
4306	u16Sel = pMixedCtx->tr.Sel;
4307	u32Limit = pMixedCtx->tr.u32Limit;
4308	u64Base = pMixedCtx->tr.u64Base;
4309	u32AccessRights = pMixedCtx->tr.Attr.u;
4310	}
4311	else
4312	{
4313	Assert(pVM->hm.s.vmx.pRealModeTSS);
4314	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4315
4316	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4317	RTGCPHYS GCPhys;
4318	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4319	AssertRCReturn(rc, rc);
4320
4321	X86DESCATTR DescAttr;
4322	DescAttr.u = 0;
4323	DescAttr.n.u1Present = 1;
4324	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4325
4326	u16Sel = 0;
4327	u32Limit = HM_VTX_TSS_SIZE;
4328	u64Base = GCPhys; /* in real-mode phys = virt. */
4329	u32AccessRights = DescAttr.u;
4330	}
4331
4332	/* Validate. */
4333	Assert(!(u16Sel & RT_BIT(2)));
4334	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4335	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4336	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4337	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4338	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4339	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4340	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4341	Assert( (u32Limit & 0xfff) == 0xfff
4342	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4343	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4344	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4345
4346	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4347	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4348	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4349	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4350
4351	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4352	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4353	}
4354
4355	/*
4356	* Guest GDTR.
4357	*/
4358	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4359	{
4360	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4361	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4362
4363	/* Validate. */
4364	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4365
4366	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4367	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4368	}
4369
4370	/*
4371	* Guest LDTR.
4372	*/
4373	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4374	{
4375	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4376	uint32_t u32Access = 0;
4377	if (!pMixedCtx->ldtr.Attr.u)
4378	u32Access = X86DESCATTR_UNUSABLE;
4379	else
4380	u32Access = pMixedCtx->ldtr.Attr.u;
4381
4382	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4383	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4384	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4385	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4386
4387	/* Validate. */
4388	if (!(u32Access & X86DESCATTR_UNUSABLE))
4389	{
4390	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4391	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4392	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4393	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4394	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4395	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4396	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4397	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4398	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4399	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4400	}
4401
4402	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4403	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4404	}
4405
4406	/*
4407	* Guest IDTR.
4408	*/
4409	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4410	{
4411	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4412	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4413
4414	/* Validate. */
4415	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4416
4417	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4418	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4419	}
4420
4421	return VINF_SUCCESS;
4422	}
4423
4424
4425	/**
4426	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4427	* areas. These MSRs will automatically be loaded to the host CPU on every
4428	* successful VM entry and stored from the host CPU on every successful VM-exit.
4429	*
4430	* This also creates/updates MSR slots for the host MSRs. The actual host
4431	* MSR values are -not- updated here for performance reasons. See
4432	* hmR0VmxSaveHostMsrs().
4433	*
4434	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4435	*
4436	* @returns VBox status code.
4437	* @param pVCpu Pointer to the VMCPU.
4438	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4439	* out-of-sync. Make sure to update the required fields
4440	* before using them.
4441	*
4442	* @remarks No-long-jump zone!!!
4443	*/
4444	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4445	{
4446	AssertPtr(pVCpu);
4447	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4448
4449	/*
4450	* MSRs that we use the auto-load/store MSR area in the VMCS.
4451	*/
4452	PVM pVM = pVCpu->CTX_SUFF(pVM);
4453	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4454	{
4455	if (pVM->hm.s.fAllow64BitGuests)
4456	{
4457	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4458	if (!HMVMX_IS_64BIT_HOST_MODE())
4459	{
4460	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4461	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4462	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4463	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4464	}
4465	# ifdef DEBUG
4466	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4467	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4468	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4469	# endif
4470	#endif
4471	}
4472	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4473	}
4474
4475	/*
4476	* Guest Sysenter MSRs.
4477	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4478	* VM-exits on WRMSRs for these MSRs.
4479	*/
4480	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4481	{
4482	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4483	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4484	}
4485
4486	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4487	{
4488	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4489	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4490	}
4491
4492	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4493	{
4494	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4495	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4496	}
4497
4498	return VINF_SUCCESS;
4499	}
4500
4501
4502	/**
4503	* Loads the guest activity state into the guest-state area in the VMCS.
4504	*
4505	* @returns VBox status code.
4506	* @param pVCpu Pointer to the VMCPU.
4507	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4508	* out-of-sync. Make sure to update the required fields
4509	* before using them.
4510	*
4511	* @remarks No-long-jump zone!!!
4512	*/
4513	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4514	{
4515	NOREF(pCtx);
4516	/** @todo See if we can make use of other states, e.g.
4517	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4518	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4519	{
4520	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4521	AssertRCReturn(rc, rc);
4522
4523	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4524	}
4525	return VINF_SUCCESS;
4526	}
4527
4528
4529	/**
4530	* Sets up the appropriate function to run guest code.
4531	*
4532	* @returns VBox status code.
4533	* @param pVCpu Pointer to the VMCPU.
4534	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4535	* out-of-sync. Make sure to update the required fields
4536	* before using them.
4537	*
4538	* @remarks No-long-jump zone!!!
4539	*/
4540	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4541	{
4542	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4543	{
4544	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4545	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4546	#endif
4547	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4548	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4549	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4550	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4551	{
4552	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4553	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4554	}
4555	#else
4556	/* 64-bit host or hybrid host. */
4557	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4558	#endif
4559	}
4560	else
4561	{
4562	/* Guest is not in long mode, use the 32-bit handler. */
4563	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4564	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4565	{
4566	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4567	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4568	}
4569	#else
4570	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4571	#endif
4572	}
4573	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4574	return VINF_SUCCESS;
4575	}
4576
4577
4578	/**
4579	* Wrapper for running the guest code in VT-x.
4580	*
4581	* @returns VBox strict status code.
4582	* @param pVM Pointer to the VM.
4583	* @param pVCpu Pointer to the VMCPU.
4584	* @param pCtx Pointer to the guest-CPU context.
4585	*
4586	* @remarks No-long-jump zone!!!
4587	*/
4588	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4589	{
4590	/*
4591	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4592	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4593	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4594	*/
4595	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4596	/** @todo Add stats for resume vs launch. */
4597	#ifdef VBOX_WITH_KERNEL_USING_XMM
4598	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4599	#else
4600	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4601	#endif
4602	}
4603
4604
4605	/**
4606	* Reports world-switch error and dumps some useful debug info.
4607	*
4608	* @param pVM Pointer to the VM.
4609	* @param pVCpu Pointer to the VMCPU.
4610	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4611	* @param pCtx Pointer to the guest-CPU context.
4612	* @param pVmxTransient Pointer to the VMX transient structure (only
4613	* exitReason updated).
4614	*/
4615	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4616	{
4617	Assert(pVM);
4618	Assert(pVCpu);
4619	Assert(pCtx);
4620	Assert(pVmxTransient);
4621	HMVMX_ASSERT_PREEMPT_SAFE();
4622
4623	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4624	switch (rcVMRun)
4625	{
4626	case VERR_VMX_INVALID_VMXON_PTR:
4627	AssertFailed();
4628	break;
4629	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4630	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4631	{
4632	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4633	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4634	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4635	AssertRC(rc);
4636
4637	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4638	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4639	Cannot do it here as we may have been long preempted. */
4640
4641	#ifdef VBOX_STRICT
4642	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4643	pVmxTransient->uExitReason));
4644	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4645	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4646	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4647	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4648	else
4649	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4650	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4651	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4652
4653	/* VMX control bits. */
4654	uint32_t u32Val;
4655	uint64_t u64Val;
4656	HMVMXHCUINTREG uHCReg;
4657	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4658	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4659	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4660	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4661	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4662	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4663	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4664	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4665	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4666	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4667	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4668	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4669	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4670	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4671	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4672	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4673	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4674	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4675	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4676	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4677	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4678	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4679	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4680	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4681	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4682	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4683	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4684	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4685	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4686	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4687	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4688	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4689	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4690	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4691	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4692	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4693	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4694	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4695	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4696	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4697	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4698	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4699
4700	/* Guest bits. */
4701	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4702	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4703	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4704	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4705	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4706	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4707	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4708	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4709
4710	/* Host bits. */
4711	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4712	Log4(("Host CR0 %#RHr\n", uHCReg));
4713	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4714	Log4(("Host CR3 %#RHr\n", uHCReg));
4715	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4716	Log4(("Host CR4 %#RHr\n", uHCReg));
4717
4718	RTGDTR HostGdtr;
4719	PCX86DESCHC pDesc;
4720	ASMGetGDTR(&HostGdtr);
4721	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4722	Log4(("Host CS %#08x\n", u32Val));
4723	if (u32Val < HostGdtr.cbGdt)
4724	{
4725	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4726	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4727	}
4728
4729	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4730	Log4(("Host DS %#08x\n", u32Val));
4731	if (u32Val < HostGdtr.cbGdt)
4732	{
4733	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4734	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4735	}
4736
4737	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4738	Log4(("Host ES %#08x\n", u32Val));
4739	if (u32Val < HostGdtr.cbGdt)
4740	{
4741	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4742	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4743	}
4744
4745	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4746	Log4(("Host FS %#08x\n", u32Val));
4747	if (u32Val < HostGdtr.cbGdt)
4748	{
4749	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4750	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4751	}
4752
4753	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4754	Log4(("Host GS %#08x\n", u32Val));
4755	if (u32Val < HostGdtr.cbGdt)
4756	{
4757	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4758	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4759	}
4760
4761	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4762	Log4(("Host SS %#08x\n", u32Val));
4763	if (u32Val < HostGdtr.cbGdt)
4764	{
4765	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4766	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4767	}
4768
4769	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4770	Log4(("Host TR %#08x\n", u32Val));
4771	if (u32Val < HostGdtr.cbGdt)
4772	{
4773	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4774	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4775	}
4776
4777	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4778	Log4(("Host TR Base %#RHv\n", uHCReg));
4779	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4780	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4781	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4782	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4783	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4784	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4785	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4786	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4787	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4788	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4789	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4790	Log4(("Host RSP %#RHv\n", uHCReg));
4791	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4792	Log4(("Host RIP %#RHv\n", uHCReg));
4793	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4794	if (HMVMX_IS_64BIT_HOST_MODE())
4795	{
4796	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4797	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4798	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4799	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4800	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4801	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4802	}
4803	# endif
4804	#endif /* VBOX_STRICT */
4805	break;
4806	}
4807
4808	default:
4809	/* Impossible */
4810	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4811	break;
4812	}
4813	NOREF(pVM); NOREF(pCtx);
4814	}
4815
4816
4817	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4818	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4819	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4820	#endif
4821	#ifdef VBOX_STRICT
4822	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4823	{
4824	switch (idxField)
4825	{
4826	case VMX_VMCS_GUEST_RIP:
4827	case VMX_VMCS_GUEST_RSP:
4828	case VMX_VMCS_GUEST_SYSENTER_EIP:
4829	case VMX_VMCS_GUEST_SYSENTER_ESP:
4830	case VMX_VMCS_GUEST_GDTR_BASE:
4831	case VMX_VMCS_GUEST_IDTR_BASE:
4832	case VMX_VMCS_GUEST_CS_BASE:
4833	case VMX_VMCS_GUEST_DS_BASE:
4834	case VMX_VMCS_GUEST_ES_BASE:
4835	case VMX_VMCS_GUEST_FS_BASE:
4836	case VMX_VMCS_GUEST_GS_BASE:
4837	case VMX_VMCS_GUEST_SS_BASE:
4838	case VMX_VMCS_GUEST_LDTR_BASE:
4839	case VMX_VMCS_GUEST_TR_BASE:
4840	case VMX_VMCS_GUEST_CR3:
4841	return true;
4842	}
4843	return false;
4844	}
4845
4846	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4847	{
4848	switch (idxField)
4849	{
4850	/* Read-only fields. */
4851	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4852	return true;
4853	}
4854	/* Remaining readable fields should also be writable. */
4855	return hmR0VmxIsValidWriteField(idxField);
4856	}
4857	#endif /* VBOX_STRICT */
4858
4859
4860	/**
4861	* Executes the specified handler in 64-bit mode.
4862	*
4863	* @returns VBox status code.
4864	* @param pVM Pointer to the VM.
4865	* @param pVCpu Pointer to the VMCPU.
4866	* @param pCtx Pointer to the guest CPU context.
4867	* @param enmOp The operation to perform.
4868	* @param cbParam Number of parameters.
4869	* @param paParam Array of 32-bit parameters.
4870	*/
4871	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4872	uint32_t *paParam)
4873	{
4874	int rc, rc2;
4875	PHMGLOBALCPUINFO pCpu;
4876	RTHCPHYS HCPhysCpuPage;
4877	RTCCUINTREG uOldEflags;
4878
4879	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4880	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4881	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4882	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4883
4884	#ifdef VBOX_STRICT
4885	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4886	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4887
4888	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4889	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4890	#endif
4891
4892	/* Disable interrupts. */
4893	uOldEflags = ASMIntDisableFlags();
4894
4895	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4896	RTCPUID idHostCpu = RTMpCpuId();
4897	CPUMR0SetLApic(pVCpu, idHostCpu);
4898	#endif
4899
4900	pCpu = HMR0GetCurrentCpu();
4901	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4902
4903	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4904	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4905
4906	/* Leave VMX Root Mode. */
4907	VMXDisable();
4908
4909	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4910
4911	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4912	CPUMSetHyperEIP(pVCpu, enmOp);
4913	for (int i = (int)cbParam - 1; i >= 0; i--)
4914	CPUMPushHyper(pVCpu, paParam[i]);
4915
4916	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4917
4918	/* Call the switcher. */
4919	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4920	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4921
4922	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4923	/* Make sure the VMX instructions don't cause #UD faults. */
4924	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4925
4926	/* Re-enter VMX Root Mode */
4927	rc2 = VMXEnable(HCPhysCpuPage);
4928	if (RT_FAILURE(rc2))
4929	{
4930	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4931	ASMSetFlags(uOldEflags);
4932	return rc2;
4933	}
4934
4935	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4936	AssertRC(rc2);
4937	Assert(!(ASMGetFlags() & X86_EFL_IF));
4938	ASMSetFlags(uOldEflags);
4939	return rc;
4940	}
4941
4942
4943	/**
4944	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4945	* supporting 64-bit guests.
4946	*
4947	* @returns VBox status code.
4948	* @param fResume Whether to VMLAUNCH or VMRESUME.
4949	* @param pCtx Pointer to the guest-CPU context.
4950	* @param pCache Pointer to the VMCS cache.
4951	* @param pVM Pointer to the VM.
4952	* @param pVCpu Pointer to the VMCPU.
4953	*/
4954	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4955	{
4956	uint32_t aParam[6];
4957	PHMGLOBALCPUINFO pCpu = NULL;
4958	RTHCPHYS HCPhysCpuPage = 0;
4959	int rc = VERR_INTERNAL_ERROR_5;
4960
4961	pCpu = HMR0GetCurrentCpu();
4962	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4963
4964	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4965	pCache->uPos = 1;
4966	pCache->interPD = PGMGetInterPaeCR3(pVM);
4967	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4968	#endif
4969
4970	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4971	pCache->TestIn.HCPhysCpuPage = 0;
4972	pCache->TestIn.HCPhysVmcs = 0;
4973	pCache->TestIn.pCache = 0;
4974	pCache->TestOut.HCPhysVmcs = 0;
4975	pCache->TestOut.pCache = 0;
4976	pCache->TestOut.pCtx = 0;
4977	pCache->TestOut.eflags = 0;
4978	#endif
4979
4980	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4981	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4982	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4983	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4984	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4985	aParam[5] = 0;
4986
4987	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4988	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4989	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4990	#endif
4991	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4992
4993	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4994	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4995	Assert(pCtx->dr[4] == 10);
4996	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4997	#endif
4998
4999	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5000	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5001	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5002	pVCpu->hm.s.vmx.HCPhysVmcs));
5003	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5004	pCache->TestOut.HCPhysVmcs));
5005	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5006	pCache->TestOut.pCache));
5007	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5008	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5009	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5010	pCache->TestOut.pCtx));
5011	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5012	#endif
5013	return rc;
5014	}
5015
5016
5017	/**
5018	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5019	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5020	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5021	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5022	*
5023	* @returns VBox status code.
5024	* @param pVM Pointer to the VM.
5025	* @param pVCpu Pointer to the VMCPU.
5026	*/
5027	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5028	{
5029	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5030	{ \
5031	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5032	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5033	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5034	++cReadFields; \
5035	}
5036
5037	AssertPtr(pVM);
5038	AssertPtr(pVCpu);
5039	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5040	uint32_t cReadFields = 0;
5041
5042	/*
5043	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5044	* and serve to indicate exceptions to the rules.
5045	*/
5046
5047	/* Guest-natural selector base fields. */
5048	#if 0
5049	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5050	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5051	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5052	#endif
5053	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5054	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5055	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5056	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5057	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5058	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5059	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5062	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5063	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5064	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5065	#if 0
5066	/* Unused natural width guest-state fields. */
5067	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5069	#endif
5070	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5071	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5072
5073	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5074	#if 0
5075	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5081	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5082	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5083	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5084	#endif
5085
5086	/* Natural width guest-state fields. */
5087	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5088	#if 0
5089	/* Currently unused field. */
5090	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5091	#endif
5092
5093	if (pVM->hm.s.fNestedPaging)
5094	{
5095	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5096	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5097	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5098	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5099	}
5100	else
5101	{
5102	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5103	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5104	}
5105
5106	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5107	return VINF_SUCCESS;
5108	}
5109
5110
5111	/**
5112	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5113	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5114	* darwin, running 64-bit guests).
5115	*
5116	* @returns VBox status code.
5117	* @param pVCpu Pointer to the VMCPU.
5118	* @param idxField The VMCS field encoding.
5119	* @param u64Val 16, 32 or 64-bit value.
5120	*/
5121	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5122	{
5123	int rc;
5124	switch (idxField)
5125	{
5126	/*
5127	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5128	*/
5129	/* 64-bit Control fields. */
5130	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5131	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5132	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5133	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5134	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5135	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5136	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5137	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5138	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5139	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5140	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5141	case VMX_VMCS64_CTRL_EPTP_FULL:
5142	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5143	/* 64-bit Guest-state fields. */
5144	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5145	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5146	case VMX_VMCS64_GUEST_PAT_FULL:
5147	case VMX_VMCS64_GUEST_EFER_FULL:
5148	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5149	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5150	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5151	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5152	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5153	/* 64-bit Host-state fields. */
5154	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5155	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5156	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5157	{
5158	rc = VMXWriteVmcs32(idxField, u64Val);
5159	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5160	break;
5161	}
5162
5163	/*
5164	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5165	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5166	*/
5167	/* Natural-width Guest-state fields. */
5168	case VMX_VMCS_GUEST_CR3:
5169	case VMX_VMCS_GUEST_ES_BASE:
5170	case VMX_VMCS_GUEST_CS_BASE:
5171	case VMX_VMCS_GUEST_SS_BASE:
5172	case VMX_VMCS_GUEST_DS_BASE:
5173	case VMX_VMCS_GUEST_FS_BASE:
5174	case VMX_VMCS_GUEST_GS_BASE:
5175	case VMX_VMCS_GUEST_LDTR_BASE:
5176	case VMX_VMCS_GUEST_TR_BASE:
5177	case VMX_VMCS_GUEST_GDTR_BASE:
5178	case VMX_VMCS_GUEST_IDTR_BASE:
5179	case VMX_VMCS_GUEST_RSP:
5180	case VMX_VMCS_GUEST_RIP:
5181	case VMX_VMCS_GUEST_SYSENTER_ESP:
5182	case VMX_VMCS_GUEST_SYSENTER_EIP:
5183	{
5184	if (!(u64Val >> 32))
5185	{
5186	/* If this field is 64-bit, VT-x will zero out the top bits. */
5187	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5188	}
5189	else
5190	{
5191	/* Assert that only the 32->64 switcher case should ever come here. */
5192	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5193	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5194	}
5195	break;
5196	}
5197
5198	default:
5199	{
5200	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5201	rc = VERR_INVALID_PARAMETER;
5202	break;
5203	}
5204	}
5205	AssertRCReturn(rc, rc);
5206	return rc;
5207	}
5208
5209
5210	/**
5211	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5212	* hosts (except darwin) for 64-bit guests.
5213	*
5214	* @param pVCpu Pointer to the VMCPU.
5215	* @param idxField The VMCS field encoding.
5216	* @param u64Val 16, 32 or 64-bit value.
5217	*/
5218	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5219	{
5220	AssertPtr(pVCpu);
5221	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5222
5223	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5224	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5225
5226	/* Make sure there are no duplicates. */
5227	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5228	{
5229	if (pCache->Write.aField[i] == idxField)
5230	{
5231	pCache->Write.aFieldVal[i] = u64Val;
5232	return VINF_SUCCESS;
5233	}
5234	}
5235
5236	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5237	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5238	pCache->Write.cValidEntries++;
5239	return VINF_SUCCESS;
5240	}
5241
5242	/* Enable later when the assembly code uses these as callbacks. */
5243	#if 0
5244	/*
5245	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5246	*
5247	* @param pVCpu Pointer to the VMCPU.
5248	* @param pCache Pointer to the VMCS cache.
5249	*
5250	* @remarks No-long-jump zone!!!
5251	*/
5252	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5253	{
5254	AssertPtr(pCache);
5255	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5256	{
5257	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5258	AssertRC(rc);
5259	}
5260	pCache->Write.cValidEntries = 0;
5261	}
5262
5263
5264	/**
5265	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5266	*
5267	* @param pVCpu Pointer to the VMCPU.
5268	* @param pCache Pointer to the VMCS cache.
5269	*
5270	* @remarks No-long-jump zone!!!
5271	*/
5272	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5273	{
5274	AssertPtr(pCache);
5275	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5276	{
5277	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5278	AssertRC(rc);
5279	}
5280	}
5281	#endif
5282	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5283
5284
5285	/**
5286	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5287	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5288	* timer.
5289	*
5290	* @returns VBox status code.
5291	* @param pVCpu Pointer to the VMCPU.
5292	*
5293	* @remarks No-long-jump zone!!!
5294	*/
5295	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5296	{
5297	int rc = VERR_INTERNAL_ERROR_5;
5298	bool fOffsettedTsc = false;
5299	PVM pVM = pVCpu->CTX_SUFF(pVM);
5300	if (pVM->hm.s.vmx.fUsePreemptTimer)
5301	{
5302	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5303
5304	/* Make sure the returned values have sane upper and lower boundaries. */
5305	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5306	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5307	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5308	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5309
5310	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5311	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5312	}
5313	else
5314	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5315
5316	if (fOffsettedTsc)
5317	{
5318	uint64_t u64CurTSC = ASMReadTSC();
5319	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5320	{
5321	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5322	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5323
5324	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5325	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5326	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5327	}
5328	else
5329	{
5330	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5331	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5332	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5333	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5334	}
5335	}
5336	else
5337	{
5338	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5339	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5340	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5341	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5342	}
5343	}
5344
5345
5346	/**
5347	* Determines if an exception is a contributory exception. Contributory
5348	* exceptions are ones which can cause double-faults. Page-fault is
5349	* intentionally not included here as it's a conditional contributory exception.
5350	*
5351	* @returns true if the exception is contributory, false otherwise.
5352	* @param uVector The exception vector.
5353	*/
5354	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5355	{
5356	switch (uVector)
5357	{
5358	case X86_XCPT_GP:
5359	case X86_XCPT_SS:
5360	case X86_XCPT_NP:
5361	case X86_XCPT_TS:
5362	case X86_XCPT_DE:
5363	return true;
5364	default:
5365	break;
5366	}
5367	return false;
5368	}
5369
5370
5371	/**
5372	* Sets an event as a pending event to be injected into the guest.
5373	*
5374	* @param pVCpu Pointer to the VMCPU.
5375	* @param u32IntInfo The VM-entry interruption-information field.
5376	* @param cbInstr The VM-entry instruction length in bytes (for software
5377	* interrupts, exceptions and privileged software
5378	* exceptions).
5379	* @param u32ErrCode The VM-entry exception error code.
5380	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5381	* page-fault.
5382	*
5383	* @remarks Statistics counter assumes this is a guest event being injected or
5384	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5385	* always incremented.
5386	*/
5387	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5388	RTGCUINTPTR GCPtrFaultAddress)
5389	{
5390	Assert(!pVCpu->hm.s.Event.fPending);
5391	pVCpu->hm.s.Event.fPending = true;
5392	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5393	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5394	pVCpu->hm.s.Event.cbInstr = cbInstr;
5395	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5396
5397	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5398	}
5399
5400
5401	/**
5402	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5403	*
5404	* @param pVCpu Pointer to the VMCPU.
5405	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5406	* out-of-sync. Make sure to update the required fields
5407	* before using them.
5408	*/
5409	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5410	{
5411	NOREF(pMixedCtx);
5412	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5413	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5414	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5415	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5416	}
5417
5418
5419	/**
5420	* Handle a condition that occurred while delivering an event through the guest
5421	* IDT.
5422	*
5423	* @returns VBox status code (informational error codes included).
5424	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5425	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5426	* continue execution of the guest which will delivery the #DF.
5427	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5428	*
5429	* @param pVCpu Pointer to the VMCPU.
5430	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5431	* out-of-sync. Make sure to update the required fields
5432	* before using them.
5433	* @param pVmxTransient Pointer to the VMX transient structure.
5434	*
5435	* @remarks No-long-jump zone!!!
5436	*/
5437	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5438	{
5439	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5440	AssertRCReturn(rc, rc);
5441	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5442	{
5443	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5444	AssertRCReturn(rc, rc);
5445
5446	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5447	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5448	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5449
5450	typedef enum
5451	{
5452	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5453	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5454	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5455	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5456	} VMXREFLECTXCPT;
5457
5458	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5459	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5460	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5461	{
5462	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5463	{
5464	enmReflect = VMXREFLECTXCPT_XCPT;
5465	#ifdef VBOX_STRICT
5466	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5467	&& uExitVector == X86_XCPT_PF)
5468	{
5469	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5470	}
5471	#endif
5472	if ( uExitVector == X86_XCPT_PF
5473	&& uIdtVector == X86_XCPT_PF)
5474	{
5475	pVmxTransient->fVectoringPF = true;
5476	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5477	}
5478	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5479	&& hmR0VmxIsContributoryXcpt(uExitVector)
5480	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5481	\|\| uIdtVector == X86_XCPT_PF))
5482	{
5483	enmReflect = VMXREFLECTXCPT_DF;
5484	}
5485	else if (uIdtVector == X86_XCPT_DF)
5486	enmReflect = VMXREFLECTXCPT_TF;
5487	}
5488	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5489	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5490	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5491	{
5492	/*
5493	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5494	* (whatever they are) as they reoccur when restarting the instruction.
5495	*/
5496	enmReflect = VMXREFLECTXCPT_XCPT;
5497	}
5498	}
5499	else
5500	{
5501	/*
5502	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5503	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5504	* original exception to the guest after handling the VM-exit.
5505	*/
5506	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5507	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5508	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5509	{
5510	enmReflect = VMXREFLECTXCPT_XCPT;
5511	}
5512	}
5513
5514	switch (enmReflect)
5515	{
5516	case VMXREFLECTXCPT_XCPT:
5517	{
5518	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5519	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5520	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5521
5522	uint32_t u32ErrCode = 0;
5523	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5524	{
5525	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5526	AssertRCReturn(rc, rc);
5527	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5528	}
5529
5530	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5531	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5532	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5533	rc = VINF_SUCCESS;
5534	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5535	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5536
5537	break;
5538	}
5539
5540	case VMXREFLECTXCPT_DF:
5541	{
5542	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5543	rc = VINF_HM_DOUBLE_FAULT;
5544	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5545	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5546
5547	break;
5548	}
5549
5550	case VMXREFLECTXCPT_TF:
5551	{
5552	rc = VINF_EM_RESET;
5553	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5554	uExitVector));
5555	break;
5556	}
5557
5558	default:
5559	Assert(rc == VINF_SUCCESS);
5560	break;
5561	}
5562	}
5563	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5564	return rc;
5565	}
5566
5567
5568	/**
5569	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5570	*
5571	* @returns VBox status code.
5572	* @param pVCpu Pointer to the VMCPU.
5573	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5574	* out-of-sync. Make sure to update the required fields
5575	* before using them.
5576	*
5577	* @remarks No-long-jump zone!!!
5578	*/
5579	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5580	{
5581	NOREF(pMixedCtx);
5582
5583	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5584	{
5585	uint32_t uVal = 0;
5586	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5587	AssertRCReturn(rc, rc);
5588
5589	uint32_t uShadow = 0;
5590	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5591	AssertRCReturn(rc, rc);
5592
5593	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5594	CPUMSetGuestCR0(pVCpu, uVal);
5595	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5596	}
5597	return VINF_SUCCESS;
5598	}
5599
5600
5601	/**
5602	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5603	*
5604	* @returns VBox status code.
5605	* @param pVCpu Pointer to the VMCPU.
5606	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5607	* out-of-sync. Make sure to update the required fields
5608	* before using them.
5609	*
5610	* @remarks No-long-jump zone!!!
5611	*/
5612	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5613	{
5614	NOREF(pMixedCtx);
5615
5616	int rc = VINF_SUCCESS;
5617	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5618	{
5619	uint32_t uVal = 0;
5620	uint32_t uShadow = 0;
5621	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5622	AssertRCReturn(rc, rc);
5623	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5624	AssertRCReturn(rc, rc);
5625
5626	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5627	CPUMSetGuestCR4(pVCpu, uVal);
5628	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5629	}
5630	return rc;
5631	}
5632
5633
5634	/**
5635	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5636	*
5637	* @returns VBox status code.
5638	* @param pVCpu Pointer to the VMCPU.
5639	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5640	* out-of-sync. Make sure to update the required fields
5641	* before using them.
5642	*
5643	* @remarks No-long-jump zone!!!
5644	*/
5645	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5646	{
5647	int rc = VINF_SUCCESS;
5648	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5649	{
5650	uint64_t u64Val = 0;
5651	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5652	AssertRCReturn(rc, rc);
5653
5654	pMixedCtx->rip = u64Val;
5655	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5656	}
5657	return rc;
5658	}
5659
5660
5661	/**
5662	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5663	*
5664	* @returns VBox status code.
5665	* @param pVCpu Pointer to the VMCPU.
5666	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5667	* out-of-sync. Make sure to update the required fields
5668	* before using them.
5669	*
5670	* @remarks No-long-jump zone!!!
5671	*/
5672	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5673	{
5674	int rc = VINF_SUCCESS;
5675	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5676	{
5677	uint64_t u64Val = 0;
5678	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5679	AssertRCReturn(rc, rc);
5680
5681	pMixedCtx->rsp = u64Val;
5682	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5683	}
5684	return rc;
5685	}
5686
5687
5688	/**
5689	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5690	*
5691	* @returns VBox status code.
5692	* @param pVCpu Pointer to the VMCPU.
5693	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5694	* out-of-sync. Make sure to update the required fields
5695	* before using them.
5696	*
5697	* @remarks No-long-jump zone!!!
5698	*/
5699	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5700	{
5701	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5702	{
5703	uint32_t uVal = 0;
5704	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5705	AssertRCReturn(rc, rc);
5706
5707	pMixedCtx->eflags.u32 = uVal;
5708	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5709	{
5710	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5711	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5712
5713	pMixedCtx->eflags.Bits.u1VM = 0;
5714	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5715	}
5716
5717	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5718	}
5719	return VINF_SUCCESS;
5720	}
5721
5722
5723	/**
5724	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5725	* guest-CPU context.
5726	*/
5727	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5728	{
5729	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5730	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5731	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5732	return rc;
5733	}
5734
5735
5736	/**
5737	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5738	* from the guest-state area in the VMCS.
5739	*
5740	* @param pVCpu Pointer to the VMCPU.
5741	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5742	* out-of-sync. Make sure to update the required fields
5743	* before using them.
5744	*
5745	* @remarks No-long-jump zone!!!
5746	*/
5747	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5748	{
5749	uint32_t uIntrState = 0;
5750	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5751	AssertRC(rc);
5752
5753	if (!uIntrState)
5754	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5755	else
5756	{
5757	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5758	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5759	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5760	AssertRC(rc);
5761	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5762	AssertRC(rc);
5763
5764	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5765	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5766	}
5767	}
5768
5769
5770	/**
5771	* Saves the guest's activity state.
5772	*
5773	* @returns VBox status code.
5774	* @param pVCpu Pointer to the VMCPU.
5775	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5776	* out-of-sync. Make sure to update the required fields
5777	* before using them.
5778	*
5779	* @remarks No-long-jump zone!!!
5780	*/
5781	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5782	{
5783	NOREF(pMixedCtx);
5784	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5785	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5786	return VINF_SUCCESS;
5787	}
5788
5789
5790	/**
5791	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5792	* the current VMCS into the guest-CPU context.
5793	*
5794	* @returns VBox status code.
5795	* @param pVCpu Pointer to the VMCPU.
5796	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5797	* out-of-sync. Make sure to update the required fields
5798	* before using them.
5799	*
5800	* @remarks No-long-jump zone!!!
5801	*/
5802	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5803	{
5804	int rc = VINF_SUCCESS;
5805	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5806	{
5807	uint32_t u32Val = 0;
5808	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5809	pMixedCtx->SysEnter.cs = u32Val;
5810	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5811	}
5812
5813	uint64_t u64Val = 0;
5814	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5815	{
5816	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5817	pMixedCtx->SysEnter.eip = u64Val;
5818	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5819	}
5820	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5821	{
5822	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5823	pMixedCtx->SysEnter.esp = u64Val;
5824	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5825	}
5826	return rc;
5827	}
5828
5829
5830	/**
5831	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5832	* the CPU back into the guest-CPU context.
5833	*
5834	* @returns VBox status code.
5835	* @param pVCpu Pointer to the VMCPU.
5836	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5837	* out-of-sync. Make sure to update the required fields
5838	* before using them.
5839	*
5840	* @remarks No-long-jump zone!!!
5841	*/
5842	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5843	{
5844	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5845	if ( HMVMX_IS_64BIT_HOST_MODE()
5846	&& pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5847	{
5848	/* We should not get preempted to a different CPU at this point while reading the MSRs. */
5849	VMMRZCallRing3Disable(pVCpu);
5850	HM_DISABLE_PREEMPT_IF_NEEDED();
5851
5852	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5853	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS))
5854	{
5855	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5856	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5857	}
5858
5859	HM_RESTORE_PREEMPT_IF_NEEDED();
5860	VMMRZCallRing3Enable(pVCpu);
5861	}
5862	else
5863	{
5864	/* Darwin 32-bit/PAE kernel or 64-bit host running 32-bit guest. */
5865	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5866	}
5867	#else /* HC_ARCH_BITS == 32 */
5868	NOREF(pMixedCtx);
5869	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5870	#endif
5871
5872	return VINF_SUCCESS;
5873	}
5874
5875
5876	/**
5877	* Saves the auto load/store'd guest MSRs from the current VMCS into
5878	* the guest-CPU context.
5879	*
5880	* @returns VBox status code.
5881	* @param pVCpu Pointer to the VMCPU.
5882	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5883	* out-of-sync. Make sure to update the required fields
5884	* before using them.
5885	*
5886	* @remarks No-long-jump zone!!!
5887	*/
5888	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5889	{
5890	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5891	return VINF_SUCCESS;
5892
5893	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5894	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5895	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5896	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5897	{
5898	switch (pMsr->u32Msr)
5899	{
5900	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5901	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5902	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5903	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5904	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5905	default:
5906	{
5907	AssertFailed();
5908	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5909	}
5910	}
5911	}
5912
5913	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5914	return VINF_SUCCESS;
5915	}
5916
5917
5918	/**
5919	* Saves the guest control registers from the current VMCS into the guest-CPU
5920	* context.
5921	*
5922	* @returns VBox status code.
5923	* @param pVCpu Pointer to the VMCPU.
5924	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5925	* out-of-sync. Make sure to update the required fields
5926	* before using them.
5927	*
5928	* @remarks No-long-jump zone!!!
5929	*/
5930	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5931	{
5932	/* Guest CR0. Guest FPU. */
5933	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5934	AssertRCReturn(rc, rc);
5935
5936	/* Guest CR4. */
5937	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5938	AssertRCReturn(rc, rc);
5939
5940	/* Guest CR2 - updated always during the world-switch or in #PF. */
5941	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5942	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5943	{
5944	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5945	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5946
5947	PVM pVM = pVCpu->CTX_SUFF(pVM);
5948	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5949	\|\| ( pVM->hm.s.fNestedPaging
5950	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5951	{
5952	uint64_t u64Val = 0;
5953	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5954	if (pMixedCtx->cr3 != u64Val)
5955	{
5956	CPUMSetGuestCR3(pVCpu, u64Val);
5957	if (VMMRZCallRing3IsEnabled(pVCpu))
5958	{
5959	PGMUpdateCR3(pVCpu, u64Val);
5960	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5961	}
5962	else
5963	{
5964	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5965	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5966	}
5967	}
5968
5969	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5970	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5971	{
5972	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5973	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5974	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5975	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5976
5977	if (VMMRZCallRing3IsEnabled(pVCpu))
5978	{
5979	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5980	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5981	}
5982	else
5983	{
5984	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5985	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5986	}
5987	}
5988	}
5989
5990	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5991	}
5992
5993	/*
5994	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5995	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5996	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5997	*
5998	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5999	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6000	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6001	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6002	*
6003	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6004	*/
6005	if (VMMRZCallRing3IsEnabled(pVCpu))
6006	{
6007	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6008	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6009
6010	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6011	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6012
6013	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6014	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6015	}
6016
6017	return rc;
6018	}
6019
6020
6021	/**
6022	* Reads a guest segment register from the current VMCS into the guest-CPU
6023	* context.
6024	*
6025	* @returns VBox status code.
6026	* @param pVCpu Pointer to the VMCPU.
6027	* @param idxSel Index of the selector in the VMCS.
6028	* @param idxLimit Index of the segment limit in the VMCS.
6029	* @param idxBase Index of the segment base in the VMCS.
6030	* @param idxAccess Index of the access rights of the segment in the VMCS.
6031	* @param pSelReg Pointer to the segment selector.
6032	*
6033	* @remarks No-long-jump zone!!!
6034	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6035	* macro as that takes care of whether to read from the VMCS cache or
6036	* not.
6037	*/
6038	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6039	PCPUMSELREG pSelReg)
6040	{
6041	NOREF(pVCpu);
6042
6043	uint32_t u32Val = 0;
6044	int rc = VMXReadVmcs32(idxSel, &u32Val);
6045	AssertRCReturn(rc, rc);
6046	pSelReg->Sel = (uint16_t)u32Val;
6047	pSelReg->ValidSel = (uint16_t)u32Val;
6048	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6049
6050	rc = VMXReadVmcs32(idxLimit, &u32Val);
6051	AssertRCReturn(rc, rc);
6052	pSelReg->u32Limit = u32Val;
6053
6054	uint64_t u64Val = 0;
6055	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6056	AssertRCReturn(rc, rc);
6057	pSelReg->u64Base = u64Val;
6058
6059	rc = VMXReadVmcs32(idxAccess, &u32Val);
6060	AssertRCReturn(rc, rc);
6061	pSelReg->Attr.u = u32Val;
6062
6063	/*
6064	* If VT-x marks the segment as unusable, most other bits remain undefined:
6065	* - For CS the L, D and G bits have meaning.
6066	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6067	* - For the remaining data segments no bits are defined.
6068	*
6069	* The present bit and the unusable bit has been observed to be set at the
6070	* same time (the selector was supposed to invalid as we started executing
6071	* a V8086 interrupt in ring-0).
6072	*
6073	* What should be important for the rest of the VBox code that the P bit is
6074	* cleared. Some of the other VBox code recognizes the unusable bit, but
6075	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6076	* safe side here, we'll strip off P and other bits we don't care about. If
6077	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6078	*
6079	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6080	*/
6081	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6082	{
6083	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6084
6085	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6086	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6087	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6088
6089	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6090	#ifdef DEBUG_bird
6091	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6092	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6093	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6094	#endif
6095	}
6096	return VINF_SUCCESS;
6097	}
6098
6099
6100	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6101	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6102	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6103	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6104	#else
6105	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6106	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6107	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6108	#endif
6109
6110
6111	/**
6112	* Saves the guest segment registers from the current VMCS into the guest-CPU
6113	* context.
6114	*
6115	* @returns VBox status code.
6116	* @param pVCpu Pointer to the VMCPU.
6117	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6118	* out-of-sync. Make sure to update the required fields
6119	* before using them.
6120	*
6121	* @remarks No-long-jump zone!!!
6122	*/
6123	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6124	{
6125	/* Guest segment registers. */
6126	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6127	{
6128	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6129	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6130	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6131	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6132	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6133	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6134	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6135
6136	/* Restore segment attributes for real-on-v86 mode hack. */
6137	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6138	{
6139	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6140	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6141	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6142	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6143	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6144	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6145	}
6146	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
6147	}
6148
6149	return VINF_SUCCESS;
6150	}
6151
6152
6153	/**
6154	* Saves the guest descriptor table registers and task register from the current
6155	* VMCS into the guest-CPU context.
6156	*
6157	* @returns VBox status code.
6158	* @param pVCpu Pointer to the VMCPU.
6159	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6160	* out-of-sync. Make sure to update the required fields
6161	* before using them.
6162	*
6163	* @remarks No-long-jump zone!!!
6164	*/
6165	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6166	{
6167	int rc = VINF_SUCCESS;
6168
6169	/* Guest LDTR. */
6170	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
6171	{
6172	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6173	AssertRCReturn(rc, rc);
6174	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
6175	}
6176
6177	/* Guest GDTR. */
6178	uint64_t u64Val = 0;
6179	uint32_t u32Val = 0;
6180	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6181	{
6182	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6183	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6184	pMixedCtx->gdtr.pGdt = u64Val;
6185	pMixedCtx->gdtr.cbGdt = u32Val;
6186	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
6187	}
6188
6189	/* Guest IDTR. */
6190	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6191	{
6192	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6193	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6194	pMixedCtx->idtr.pIdt = u64Val;
6195	pMixedCtx->idtr.cbIdt = u32Val;
6196	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
6197	}
6198
6199	/* Guest TR. */
6200	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6201	{
6202	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6203	AssertRCReturn(rc, rc);
6204
6205	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6206	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6207	{
6208	rc = VMXLOCAL_READ_SEG(TR, tr);
6209	AssertRCReturn(rc, rc);
6210	}
6211	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
6212	}
6213	return rc;
6214	}
6215
6216	#undef VMXLOCAL_READ_SEG
6217
6218
6219	/**
6220	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6221	* context.
6222	*
6223	* @returns VBox status code.
6224	* @param pVCpu Pointer to the VMCPU.
6225	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6226	* out-of-sync. Make sure to update the required fields
6227	* before using them.
6228	*
6229	* @remarks No-long-jump zone!!!
6230	*/
6231	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6232	{
6233	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6234	{
6235	if (!pVCpu->hm.s.fUsingHyperDR7)
6236	{
6237	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6238	uint32_t u32Val;
6239	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6240	pMixedCtx->dr[7] = u32Val;
6241	}
6242
6243	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
6244	}
6245	return VINF_SUCCESS;
6246	}
6247
6248
6249	/**
6250	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6251	*
6252	* @returns VBox status code.
6253	* @param pVCpu Pointer to the VMCPU.
6254	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6255	* out-of-sync. Make sure to update the required fields
6256	* before using them.
6257	*
6258	* @remarks No-long-jump zone!!!
6259	*/
6260	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6261	{
6262	NOREF(pMixedCtx);
6263
6264	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6265	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
6266	return VINF_SUCCESS;
6267	}
6268
6269
6270	/**
6271	* Saves the entire guest state from the currently active VMCS into the
6272	* guest-CPU context. This essentially VMREADs all guest-data.
6273	*
6274	* @returns VBox status code.
6275	* @param pVCpu Pointer to the VMCPU.
6276	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6277	* out-of-sync. Make sure to update the required fields
6278	* before using them.
6279	*/
6280	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6281	{
6282	Assert(pVCpu);
6283	Assert(pMixedCtx);
6284
6285	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6286	return VINF_SUCCESS;
6287
6288	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6289	again on the ring-3 callback path, there is no real need to. */
6290	if (VMMRZCallRing3IsEnabled(pVCpu))
6291	VMMR0LogFlushDisable(pVCpu);
6292	else
6293	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6294	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6295
6296	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6297	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6298
6299	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6300	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6301
6302	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6303	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6304
6305	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6306	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6307
6308	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6309	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6310
6311	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6312	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6313
6314	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6315	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6316
6317	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6318	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6319
6320	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6321	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6322
6323	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6324	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6325
6326	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6327	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6328
6329	if (VMMRZCallRing3IsEnabled(pVCpu))
6330	VMMR0LogFlushEnable(pVCpu);
6331
6332	return rc;
6333	}
6334
6335
6336	/**
6337	* Check per-VM and per-VCPU force flag actions that require us to go back to
6338	* ring-3 for one reason or another.
6339	*
6340	* @returns VBox status code (information status code included).
6341	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6342	* ring-3.
6343	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6344	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6345	* interrupts)
6346	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6347	* all EMTs to be in ring-3.
6348	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6349	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6350	* to the EM loop.
6351	*
6352	* @param pVM Pointer to the VM.
6353	* @param pVCpu Pointer to the VMCPU.
6354	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6355	* out-of-sync. Make sure to update the required fields
6356	* before using them.
6357	*/
6358	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6359	{
6360	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6361
6362	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6363	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6364	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6365	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6366	{
6367	/* We need the control registers now, make sure the guest-CPU context is updated. */
6368	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6369	AssertRCReturn(rc3, rc3);
6370
6371	/* Pending HM CR3 sync. */
6372	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6373	{
6374	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6375	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6376	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6377	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6378	}
6379
6380	/* Pending HM PAE PDPEs. */
6381	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6382	{
6383	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6384	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6385	}
6386
6387	/* Pending PGM C3 sync. */
6388	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6389	{
6390	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6391	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6392	if (rc2 != VINF_SUCCESS)
6393	{
6394	AssertRC(rc2);
6395	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6396	return rc2;
6397	}
6398	}
6399
6400	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6401	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6402	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6403	{
6404	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6405	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6406	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6407	return rc2;
6408	}
6409
6410	/* Pending VM request packets, such as hardware interrupts. */
6411	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6412	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6413	{
6414	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6415	return VINF_EM_PENDING_REQUEST;
6416	}
6417
6418	/* Pending PGM pool flushes. */
6419	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6420	{
6421	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6422	return VINF_PGM_POOL_FLUSH_PENDING;
6423	}
6424
6425	/* Pending DMA requests. */
6426	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6427	{
6428	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6429	return VINF_EM_RAW_TO_R3;
6430	}
6431	}
6432
6433	return VINF_SUCCESS;
6434	}
6435
6436
6437	/**
6438	* Converts any TRPM trap into a pending HM event. This is typically used when
6439	* entering from ring-3 (not longjmp returns).
6440	*
6441	* @param pVCpu Pointer to the VMCPU.
6442	*/
6443	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6444	{
6445	Assert(TRPMHasTrap(pVCpu));
6446	Assert(!pVCpu->hm.s.Event.fPending);
6447
6448	uint8_t uVector;
6449	TRPMEVENT enmTrpmEvent;
6450	RTGCUINT uErrCode;
6451	RTGCUINTPTR GCPtrFaultAddress;
6452	uint8_t cbInstr;
6453
6454	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6455	AssertRC(rc);
6456
6457	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6458	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6459	if (enmTrpmEvent == TRPM_TRAP)
6460	{
6461	switch (uVector)
6462	{
6463	case X86_XCPT_BP:
6464	case X86_XCPT_OF:
6465	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6466	break;
6467
6468	case X86_XCPT_PF:
6469	case X86_XCPT_DF:
6470	case X86_XCPT_TS:
6471	case X86_XCPT_NP:
6472	case X86_XCPT_SS:
6473	case X86_XCPT_GP:
6474	case X86_XCPT_AC:
6475	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6476	/* no break! */
6477	default:
6478	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6479	break;
6480	}
6481	}
6482	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6483	{
6484	if (uVector == X86_XCPT_NMI)
6485	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6486	else
6487	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6488	}
6489	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6490	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6491	else
6492	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6493
6494	rc = TRPMResetTrap(pVCpu);
6495	AssertRC(rc);
6496	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6497	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6498
6499	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6500	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6501	}
6502
6503
6504	/**
6505	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6506	* VT-x to execute any instruction.
6507	*
6508	* @param pvCpu Pointer to the VMCPU.
6509	*/
6510	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6511	{
6512	Assert(pVCpu->hm.s.Event.fPending);
6513
6514	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6515	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6516	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6517	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6518
6519	/* If a trap was already pending, we did something wrong! */
6520	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6521
6522	TRPMEVENT enmTrapType;
6523	switch (uVectorType)
6524	{
6525	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6526	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6527	enmTrapType = TRPM_HARDWARE_INT;
6528	break;
6529
6530	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6531	enmTrapType = TRPM_SOFTWARE_INT;
6532	break;
6533
6534	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6535	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6536	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6537	enmTrapType = TRPM_TRAP;
6538	break;
6539
6540	default:
6541	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6542	enmTrapType = TRPM_32BIT_HACK;
6543	break;
6544	}
6545
6546	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6547
6548	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6549	AssertRC(rc);
6550
6551	if (fErrorCodeValid)
6552	TRPMSetErrorCode(pVCpu, uErrorCode);
6553
6554	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6555	&& uVector == X86_XCPT_PF)
6556	{
6557	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6558	}
6559	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6560	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6561	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6562	{
6563	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6564	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6565	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6566	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6567	}
6568	pVCpu->hm.s.Event.fPending = false;
6569	}
6570
6571
6572	/**
6573	* Does the necessary state syncing before returning to ring-3 for any reason
6574	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6575	*
6576	* @returns VBox status code.
6577	* @param pVM Pointer to the VM.
6578	* @param pVCpu Pointer to the VMCPU.
6579	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6580	* be out-of-sync. Make sure to update the required
6581	* fields before using them.
6582	* @param fSaveGuestState Whether to save the guest state or not.
6583	*
6584	* @remarks If you modify code here, make sure to check whether
6585	* hmR0VmxCallRing3Callback() needs to be updated too!!!
6586	* @remarks No-long-jmp zone!!!
6587	*/
6588	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6589	{
6590	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6591	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6592
6593	RTCPUID idCpu = RTMpCpuId();
6594	Log4Func(("HostCpuId=%u\n", idCpu));
6595
6596	/* Save the guest state if necessary. */
6597	if ( fSaveGuestState
6598	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6599	{
6600	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6601	AssertRCReturn(rc, rc);
6602	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6603	}
6604
6605	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6606	if (CPUMIsGuestFPUStateActive(pVCpu))
6607	{
6608	/* We shouldn't reload CR0 without saving it first. */
6609	if (!fSaveGuestState)
6610	{
6611	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6612	AssertRCReturn(rc, rc);
6613	}
6614	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6615	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6616	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6617	}
6618
6619	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6620	#ifdef VBOX_STRICT
6621	if (CPUMIsHyperDebugStateActive(pVCpu))
6622	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6623	#endif
6624	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6625	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6626	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6627	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6628
6629	#if HC_ARCH_BITS == 64
6630	/* Restore host-state bits that VT-x only restores partially. */
6631	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6632	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6633	{
6634	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6635	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6636	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6637	}
6638	#endif
6639
6640	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6641	/* Restore the host MSRs as we're leaving VT-x context. */
6642	if ( HMVMX_IS_64BIT_HOST_MODE()
6643	&& pVM->hm.s.fAllow64BitGuests
6644	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6645	{
6646	/* We shouldn't reload the guest MSRs without saving it first. */
6647	if (!fSaveGuestState)
6648	{
6649	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6650	AssertRCReturn(rc, rc);
6651	}
6652	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS);
6653	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6654	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6655	}
6656	#endif
6657
6658	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6659	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6660	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6661	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6662	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6663	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6664	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6665	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6666
6667	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6668
6669	/** @todo This partially defeats the purpose of having preemption hooks.
6670	* The problem is, deregistering the hooks should be moved to a place that
6671	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6672	* context.
6673	*/
6674	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6675	{
6676	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6677	AssertRCReturn(rc, rc);
6678
6679	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6680	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6681	}
6682	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6683	NOREF(idCpu);
6684
6685	return VINF_SUCCESS;
6686	}
6687
6688
6689	/**
6690	* Leaves the VT-x session.
6691	*
6692	* @returns VBox status code.
6693	* @param pVM Pointer to the VM.
6694	* @param pVCpu Pointer to the VMCPU.
6695	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6696	* out-of-sync. Make sure to update the required fields
6697	* before using them.
6698	*
6699	* @remarks No-long-jmp zone!!!
6700	*/
6701	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6702	{
6703	HM_DISABLE_PREEMPT_IF_NEEDED();
6704	HMVMX_ASSERT_CPU_SAFE();
6705	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6706	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6707
6708	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6709	and done this from the VMXR0ThreadCtxCallback(). */
6710	if (!pVCpu->hm.s.fLeaveDone)
6711	{
6712	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6713	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6714	pVCpu->hm.s.fLeaveDone = true;
6715	}
6716
6717	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6718	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6719	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6720	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6721	VMMR0ThreadCtxHooksDeregister(pVCpu);
6722
6723	/* Leave HM context. This takes care of local init (term). */
6724	int rc = HMR0LeaveCpu(pVCpu);
6725
6726	HM_RESTORE_PREEMPT_IF_NEEDED();
6727
6728	return rc;
6729	}
6730
6731
6732	/**
6733	* Does the necessary state syncing before doing a longjmp to ring-3.
6734	*
6735	* @returns VBox status code.
6736	* @param pVM Pointer to the VM.
6737	* @param pVCpu Pointer to the VMCPU.
6738	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6739	* out-of-sync. Make sure to update the required fields
6740	* before using them.
6741	*
6742	* @remarks No-long-jmp zone!!!
6743	*/
6744	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6745	{
6746	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6747	}
6748
6749
6750	/**
6751	* Take necessary actions before going back to ring-3.
6752	*
6753	* An action requires us to go back to ring-3. This function does the necessary
6754	* steps before we can safely return to ring-3. This is not the same as longjmps
6755	* to ring-3, this is voluntary and prepares the guest so it may continue
6756	* executing outside HM (recompiler/IEM).
6757	*
6758	* @returns VBox status code.
6759	* @param pVM Pointer to the VM.
6760	* @param pVCpu Pointer to the VMCPU.
6761	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6762	* out-of-sync. Make sure to update the required fields
6763	* before using them.
6764	* @param rcExit The reason for exiting to ring-3. Can be
6765	* VINF_VMM_UNKNOWN_RING3_CALL.
6766	*/
6767	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6768	{
6769	Assert(pVM);
6770	Assert(pVCpu);
6771	Assert(pMixedCtx);
6772	HMVMX_ASSERT_PREEMPT_SAFE();
6773
6774	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6775	{
6776	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6777	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6778	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6779	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6780	}
6781
6782	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6783	VMMRZCallRing3Disable(pVCpu);
6784	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6785
6786	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6787	if (pVCpu->hm.s.Event.fPending)
6788	{
6789	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6790	Assert(!pVCpu->hm.s.Event.fPending);
6791	}
6792
6793	/* Save guest state and restore host state bits. */
6794	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6795	AssertRCReturn(rc, rc);
6796	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6797
6798	/* Sync recompiler state. */
6799	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6800	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6801	\| CPUM_CHANGED_LDTR
6802	\| CPUM_CHANGED_GDTR
6803	\| CPUM_CHANGED_IDTR
6804	\| CPUM_CHANGED_TR
6805	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6806	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6807	if ( pVM->hm.s.fNestedPaging
6808	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6809	{
6810	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6811	}
6812
6813	Assert(!pVCpu->hm.s.fClearTrapFlag);
6814
6815	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6816	if (rcExit != VINF_EM_RAW_INTERRUPT)
6817	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6818
6819	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6820
6821	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6822	VMMRZCallRing3RemoveNotification(pVCpu);
6823	VMMRZCallRing3Enable(pVCpu);
6824
6825	return rc;
6826	}
6827
6828
6829	/**
6830	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6831	* longjump to ring-3 and possibly get preempted.
6832	*
6833	* @returns VBox status code.
6834	* @param pVCpu Pointer to the VMCPU.
6835	* @param enmOperation The operation causing the ring-3 longjump.
6836	* @param pvUser Opaque pointer to the guest-CPU context. The data
6837	* may be out-of-sync. Make sure to update the required
6838	* fields before using them.
6839	*
6840	* @remarks If you modify code here, make sure to check whether
6841	* hmR0VmxLeave() needs to be updated too!!!
6842	*/
6843	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6844	{
6845	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6846	{
6847	/* If anything here asserts or fails, good luck. */
6848	VMMRZCallRing3RemoveNotification(pVCpu);
6849	VMMRZCallRing3Disable(pVCpu);
6850	HM_DISABLE_PREEMPT_IF_NEEDED();
6851
6852	PVM pVM = pVCpu->CTX_SUFF(pVM);
6853	if (CPUMIsGuestFPUStateActive(pVCpu))
6854	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6855
6856	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6857
6858	#if HC_ARCH_BITS == 64
6859	/* Restore host-state bits that VT-x only restores partially. */
6860	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6861	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6862	{
6863	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6864	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6865	}
6866	#endif
6867
6868	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6869	/* Restore the host MSRs as we're leaving VT-x context. */
6870	if ( HMVMX_IS_64BIT_HOST_MODE()
6871	&& pVM->hm.s.fAllow64BitGuests
6872	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6873	{
6874	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6875	}
6876	#endif
6877	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6878	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6879	{
6880	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6881	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6882	}
6883
6884	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6885	VMMR0ThreadCtxHooksDeregister(pVCpu);
6886
6887	HMR0LeaveCpu(pVCpu);
6888	HM_RESTORE_PREEMPT_IF_NEEDED();
6889	return VINF_SUCCESS;
6890	}
6891
6892	Assert(pVCpu);
6893	Assert(pvUser);
6894	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6895	HMVMX_ASSERT_PREEMPT_SAFE();
6896
6897	VMMRZCallRing3Disable(pVCpu);
6898	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6899
6900	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6901	enmOperation));
6902
6903	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6904	AssertRCReturn(rc, rc);
6905
6906	VMMRZCallRing3Enable(pVCpu);
6907	return VINF_SUCCESS;
6908	}
6909
6910
6911	/**
6912	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6913	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6914	*
6915	* @param pVCpu Pointer to the VMCPU.
6916	*/
6917	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6918	{
6919	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6920	{
6921	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6922	{
6923	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6924	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6925	AssertRC(rc);
6926	}
6927	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6928	}
6929
6930
6931	/**
6932	* Evaluates the event to be delivered to the guest and sets it as the pending
6933	* event.
6934	*
6935	* @param pVCpu Pointer to the VMCPU.
6936	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6937	* out-of-sync. Make sure to update the required fields
6938	* before using them.
6939	*/
6940	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6941	{
6942	Assert(!pVCpu->hm.s.Event.fPending);
6943
6944	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6945	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6946	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6947	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6948
6949	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6950	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6951	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6952	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6953	Assert(!TRPMHasTrap(pVCpu));
6954
6955	/** @todo SMI. SMIs take priority over NMIs. */
6956	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6957	{
6958	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6959	if ( !fBlockMovSS
6960	&& !fBlockSti)
6961	{
6962	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6963	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6964	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6965	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6966
6967	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6968	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6969	}
6970	else
6971	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6972	}
6973	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6974	&& !pVCpu->hm.s.fSingleInstruction)
6975	{
6976	/*
6977	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6978	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6979	* evaluated here and not set as pending, solely based on the force-flags.
6980	*/
6981	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6982	AssertRC(rc);
6983	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6984	if ( !fBlockInt
6985	&& !fBlockSti
6986	&& !fBlockMovSS)
6987	{
6988	uint8_t u8Interrupt;
6989	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6990	if (RT_SUCCESS(rc))
6991	{
6992	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6993	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6994	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6995
6996	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6997	}
6998	else
6999	{
7000	/** @todo Does this actually happen? If not turn it into an assertion. */
7001	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7002	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7003	}
7004	}
7005	else
7006	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7007	}
7008	}
7009
7010
7011	/**
7012	* Injects any pending events into the guest if the guest is in a state to
7013	* receive them.
7014	*
7015	* @returns VBox status code (informational status codes included).
7016	* @param pVCpu Pointer to the VMCPU.
7017	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7018	* out-of-sync. Make sure to update the required fields
7019	* before using them.
7020	*/
7021	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7022	{
7023	HMVMX_ASSERT_PREEMPT_SAFE();
7024	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7025
7026	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7027	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7028	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7029	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7030
7031	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
7032	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7033	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7034	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7035	Assert(!TRPMHasTrap(pVCpu));
7036
7037	int rc = VINF_SUCCESS;
7038	if (pVCpu->hm.s.Event.fPending)
7039	{
7040	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7041	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7042	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7043	{
7044	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7045	AssertRCReturn(rc, rc);
7046	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7047	Assert(!fBlockInt);
7048	Assert(!fBlockSti);
7049	Assert(!fBlockMovSS);
7050	}
7051	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7052	{
7053	Assert(!fBlockSti);
7054	Assert(!fBlockMovSS);
7055	}
7056	#endif
7057	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7058	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7059	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7060	AssertRCReturn(rc, rc);
7061
7062	/* Update the interruptibility-state as it could have been changed by
7063	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7064	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7065	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7066
7067	#ifdef VBOX_WITH_STATISTICS
7068	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7069	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7070	else
7071	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7072	#endif
7073	}
7074
7075	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7076	if ( !pVCpu->hm.s.fSingleInstruction
7077	&& !DBGFIsStepping(pVCpu))
7078	{
7079	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7080	AssertRCReturn(rc2, rc2);
7081	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7082	{
7083	/*
7084	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7085	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7086	* See Intel spec. 27.3.4 "Saving Non-Register State".
7087	*/
7088	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7089	AssertRCReturn(rc2, rc2);
7090	}
7091	}
7092	else
7093	{
7094	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
7095	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
7096	uIntrState = 0;
7097	}
7098
7099	/*
7100	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7101	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7102	*/
7103	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7104	AssertRC(rc2);
7105
7106	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7107	NOREF(fBlockMovSS); NOREF(fBlockSti);
7108	return rc;
7109	}
7110
7111
7112	/**
7113	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7114	*
7115	* @param pVCpu Pointer to the VMCPU.
7116	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7117	* out-of-sync. Make sure to update the required fields
7118	* before using them.
7119	*/
7120	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7121	{
7122	NOREF(pMixedCtx);
7123	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7124	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7125	}
7126
7127
7128	/**
7129	* Injects a double-fault (#DF) exception into the VM.
7130	*
7131	* @returns VBox status code (informational status code included).
7132	* @param pVCpu Pointer to the VMCPU.
7133	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7134	* out-of-sync. Make sure to update the required fields
7135	* before using them.
7136	*/
7137	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7138	{
7139	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7140	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7141	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7142	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7143	puIntrState);
7144	}
7145
7146
7147	/**
7148	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7149	*
7150	* @param pVCpu Pointer to the VMCPU.
7151	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7152	* out-of-sync. Make sure to update the required fields
7153	* before using them.
7154	*/
7155	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7156	{
7157	NOREF(pMixedCtx);
7158	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7159	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7160	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7161	}
7162
7163
7164	/**
7165	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7166	*
7167	* @param pVCpu Pointer to the VMCPU.
7168	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7169	* out-of-sync. Make sure to update the required fields
7170	* before using them.
7171	* @param cbInstr The value of RIP that is to be pushed on the guest
7172	* stack.
7173	*/
7174	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7175	{
7176	NOREF(pMixedCtx);
7177	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7178	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7179	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7180	}
7181
7182
7183	/**
7184	* Injects a general-protection (#GP) fault into the VM.
7185	*
7186	* @returns VBox status code (informational status code included).
7187	* @param pVCpu Pointer to the VMCPU.
7188	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7189	* out-of-sync. Make sure to update the required fields
7190	* before using them.
7191	* @param u32ErrorCode The error code associated with the #GP.
7192	*/
7193	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7194	uint32_t *puIntrState)
7195	{
7196	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7197	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7198	if (fErrorCodeValid)
7199	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7200	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7201	puIntrState);
7202	}
7203
7204
7205	/**
7206	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7207	*
7208	* @param pVCpu Pointer to the VMCPU.
7209	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7210	* out-of-sync. Make sure to update the required fields
7211	* before using them.
7212	* @param uVector The software interrupt vector number.
7213	* @param cbInstr The value of RIP that is to be pushed on the guest
7214	* stack.
7215	*/
7216	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7217	{
7218	NOREF(pMixedCtx);
7219	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7220	if ( uVector == X86_XCPT_BP
7221	\|\| uVector == X86_XCPT_OF)
7222	{
7223	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7224	}
7225	else
7226	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7227	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7228	}
7229
7230
7231	/**
7232	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7233	* stack.
7234	*
7235	* @returns VBox status code (information status code included).
7236	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7237	* @param pVM Pointer to the VM.
7238	* @param pMixedCtx Pointer to the guest-CPU context.
7239	* @param uValue The value to push to the guest stack.
7240	*/
7241	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7242	{
7243	/*
7244	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7245	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7246	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7247	*/
7248	if (pMixedCtx->sp == 1)
7249	return VINF_EM_RESET;
7250	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7251	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7252	AssertRCReturn(rc, rc);
7253	return rc;
7254	}
7255
7256
7257	/**
7258	* Injects an event into the guest upon VM-entry by updating the relevant fields
7259	* in the VM-entry area in the VMCS.
7260	*
7261	* @returns VBox status code (informational error codes included).
7262	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7263	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7264	*
7265	* @param pVCpu Pointer to the VMCPU.
7266	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7267	* be out-of-sync. Make sure to update the required
7268	* fields before using them.
7269	* @param u64IntInfo The VM-entry interruption-information field.
7270	* @param cbInstr The VM-entry instruction length in bytes (for
7271	* software interrupts, exceptions and privileged
7272	* software exceptions).
7273	* @param u32ErrCode The VM-entry exception error code.
7274	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7275	* @param puIntrState Pointer to the current guest interruptibility-state.
7276	* This interruptibility-state will be updated if
7277	* necessary. This cannot not be NULL.
7278	*
7279	* @remarks Requires CR0!
7280	* @remarks No-long-jump zone!!!
7281	*/
7282	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7283	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7284	{
7285	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7286	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7287	Assert(puIntrState);
7288	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7289
7290	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7291	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7292
7293	#ifdef VBOX_STRICT
7294	/* Validate the error-code-valid bit for hardware exceptions. */
7295	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7296	{
7297	switch (uVector)
7298	{
7299	case X86_XCPT_PF:
7300	case X86_XCPT_DF:
7301	case X86_XCPT_TS:
7302	case X86_XCPT_NP:
7303	case X86_XCPT_SS:
7304	case X86_XCPT_GP:
7305	case X86_XCPT_AC:
7306	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7307	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7308	/* fallthru */
7309	default:
7310	break;
7311	}
7312	}
7313	#endif
7314
7315	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7316	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7317	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7318
7319	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7320
7321	/* We require CR0 to check if the guest is in real-mode. */
7322	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7323	AssertRCReturn(rc, rc);
7324
7325	/*
7326	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7327	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7328	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7329	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7330	*/
7331	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7332	{
7333	PVM pVM = pVCpu->CTX_SUFF(pVM);
7334	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7335	{
7336	Assert(PDMVmmDevHeapIsEnabled(pVM));
7337	Assert(pVM->hm.s.vmx.pRealModeTSS);
7338
7339	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7340	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7341	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7342	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7343	AssertRCReturn(rc, rc);
7344	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7345
7346	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7347	const size_t cbIdtEntry = sizeof(X86IDTR16);
7348	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7349	{
7350	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7351	if (uVector == X86_XCPT_DF)
7352	return VINF_EM_RESET;
7353	else if (uVector == X86_XCPT_GP)
7354	{
7355	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7356	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7357	}
7358
7359	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7360	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7361	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7362	}
7363
7364	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7365	uint16_t uGuestIp = pMixedCtx->ip;
7366	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7367	{
7368	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7369	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7370	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7371	}
7372	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7373	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7374
7375	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7376	X86IDTR16 IdtEntry;
7377	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7378	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7379	AssertRCReturn(rc, rc);
7380
7381	/* Construct the stack frame for the interrupt/exception handler. */
7382	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7383	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7384	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7385	AssertRCReturn(rc, rc);
7386
7387	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7388	if (rc == VINF_SUCCESS)
7389	{
7390	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7391	pMixedCtx->rip = IdtEntry.offSel;
7392	pMixedCtx->cs.Sel = IdtEntry.uSel;
7393	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7394	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7395	&& uVector == X86_XCPT_PF)
7396	{
7397	pMixedCtx->cr2 = GCPtrFaultAddress;
7398	}
7399
7400	/* If any other guest-state bits are changed here, make sure to update
7401	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7402	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7403	\| HM_CHANGED_GUEST_RIP
7404	\| HM_CHANGED_GUEST_RFLAGS
7405	\| HM_CHANGED_GUEST_RSP);
7406
7407	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7408	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7409	{
7410	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7411	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7412	Log4(("Clearing inhibition due to STI.\n"));
7413	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7414	}
7415	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7416
7417	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7418	it, if we are returning to ring-3 before executing guest code. */
7419	pVCpu->hm.s.Event.fPending = false;
7420	}
7421	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7422	return rc;
7423	}
7424	else
7425	{
7426	/*
7427	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7428	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7429	*/
7430	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7431	}
7432	}
7433
7434	/* Validate. */
7435	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7436	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7437	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7438
7439	/* Inject. */
7440	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7441	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7442	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7443	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7444
7445	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7446	&& uVector == X86_XCPT_PF)
7447	{
7448	pMixedCtx->cr2 = GCPtrFaultAddress;
7449	}
7450
7451	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7452	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7453
7454	AssertRCReturn(rc, rc);
7455	return rc;
7456	}
7457
7458
7459	/**
7460	* Clears the interrupt-window exiting control in the VMCS and if necessary
7461	* clears the current event in the VMCS as well.
7462	*
7463	* @returns VBox status code.
7464	* @param pVCpu Pointer to the VMCPU.
7465	*
7466	* @remarks Use this function only to clear events that have not yet been
7467	* delivered to the guest but are injected in the VMCS!
7468	* @remarks No-long-jump zone!!!
7469	*/
7470	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7471	{
7472	int rc;
7473	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7474
7475	/* Clear interrupt-window exiting control. */
7476	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7477	{
7478	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7479	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7480	AssertRC(rc);
7481	}
7482
7483	if (!pVCpu->hm.s.Event.fPending)
7484	return;
7485
7486	#ifdef VBOX_STRICT
7487	uint32_t u32EntryInfo;
7488	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7489	AssertRC(rc);
7490	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7491	#endif
7492
7493	/* Clear the entry-interruption field (including the valid bit). */
7494	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7495	AssertRC(rc);
7496
7497	/* Clear the pending debug exception field. */
7498	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7499	AssertRC(rc);
7500	}
7501
7502
7503	/**
7504	* Enters the VT-x session.
7505	*
7506	* @returns VBox status code.
7507	* @param pVM Pointer to the VM.
7508	* @param pVCpu Pointer to the VMCPU.
7509	* @param pCpu Pointer to the CPU info struct.
7510	*/
7511	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7512	{
7513	AssertPtr(pVM);
7514	AssertPtr(pVCpu);
7515	Assert(pVM->hm.s.vmx.fSupported);
7516	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7517	NOREF(pCpu); NOREF(pVM);
7518
7519	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7520	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7521
7522	#ifdef VBOX_STRICT
7523	/* Make sure we're in VMX root mode. */
7524	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7525	if (!(u32HostCR4 & X86_CR4_VMXE))
7526	{
7527	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7528	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7529	}
7530	#endif
7531
7532	/*
7533	* Load the VCPU's VMCS as the current (and active) one.
7534	*/
7535	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7536	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7537	if (RT_FAILURE(rc))
7538	return rc;
7539
7540	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7541	pVCpu->hm.s.fLeaveDone = false;
7542	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7543
7544	return VINF_SUCCESS;
7545	}
7546
7547
7548	/**
7549	* The thread-context callback (only on platforms which support it).
7550	*
7551	* @param enmEvent The thread-context event.
7552	* @param pVCpu Pointer to the VMCPU.
7553	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7554	* @thread EMT(pVCpu)
7555	*/
7556	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7557	{
7558	NOREF(fGlobalInit);
7559
7560	switch (enmEvent)
7561	{
7562	case RTTHREADCTXEVENT_PREEMPTING:
7563	{
7564	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7565	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7566	VMCPU_ASSERT_EMT(pVCpu);
7567
7568	PVM pVM = pVCpu->CTX_SUFF(pVM);
7569	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7570
7571	/* No longjmps (logger flushes, locks) in this fragile context. */
7572	VMMRZCallRing3Disable(pVCpu);
7573	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7574
7575	/*
7576	* Restore host-state (FPU, debug etc.)
7577	*/
7578	if (!pVCpu->hm.s.fLeaveDone)
7579	{
7580	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7581	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7582	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7583	pVCpu->hm.s.fLeaveDone = true;
7584	}
7585
7586	/* Leave HM context, takes care of local init (term). */
7587	int rc = HMR0LeaveCpu(pVCpu);
7588	AssertRC(rc); NOREF(rc);
7589
7590	/* Restore longjmp state. */
7591	VMMRZCallRing3Enable(pVCpu);
7592	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7593	break;
7594	}
7595
7596	case RTTHREADCTXEVENT_RESUMED:
7597	{
7598	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7599	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7600	VMCPU_ASSERT_EMT(pVCpu);
7601
7602	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7603	VMMRZCallRing3Disable(pVCpu);
7604	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7605
7606	/* Initialize the bare minimum state required for HM. This takes care of
7607	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7608	int rc = HMR0EnterCpu(pVCpu);
7609	AssertRC(rc);
7610	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7611
7612	/* Load the active VMCS as the current one. */
7613	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7614	{
7615	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7616	AssertRC(rc); NOREF(rc);
7617	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7618	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7619	}
7620	pVCpu->hm.s.fLeaveDone = false;
7621
7622	/* Restore longjmp state. */
7623	VMMRZCallRing3Enable(pVCpu);
7624	break;
7625	}
7626
7627	default:
7628	break;
7629	}
7630	}
7631
7632
7633	/**
7634	* Saves the host state in the VMCS host-state.
7635	* Sets up the VM-exit MSR-load area.
7636	*
7637	* The CPU state will be loaded from these fields on every successful VM-exit.
7638	*
7639	* @returns VBox status code.
7640	* @param pVM Pointer to the VM.
7641	* @param pVCpu Pointer to the VMCPU.
7642	*
7643	* @remarks No-long-jump zone!!!
7644	*/
7645	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7646	{
7647	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7648
7649	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7650	return VINF_SUCCESS;
7651
7652	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7653	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7654
7655	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7656	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7657
7658	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7659	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7660
7661	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7662	return rc;
7663	}
7664
7665
7666	/**
7667	* Saves the host state in the VMCS host-state.
7668	*
7669	* @returns VBox status code.
7670	* @param pVM Pointer to the VM.
7671	* @param pVCpu Pointer to the VMCPU.
7672	*
7673	* @remarks No-long-jump zone!!!
7674	*/
7675	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7676	{
7677	AssertPtr(pVM);
7678	AssertPtr(pVCpu);
7679
7680	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7681
7682	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7683	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7684	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7685	return hmR0VmxSaveHostState(pVM, pVCpu);
7686	}
7687
7688
7689	/**
7690	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7691	* loaded from these fields on every successful VM-entry.
7692	*
7693	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7694	* Sets up the VM-entry controls.
7695	* Sets up the appropriate VMX non-root function to execute guest code based on
7696	* the guest CPU mode.
7697	*
7698	* @returns VBox status code.
7699	* @param pVM Pointer to the VM.
7700	* @param pVCpu Pointer to the VMCPU.
7701	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7702	* out-of-sync. Make sure to update the required fields
7703	* before using them.
7704	*
7705	* @remarks No-long-jump zone!!!
7706	*/
7707	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7708	{
7709	AssertPtr(pVM);
7710	AssertPtr(pVCpu);
7711	AssertPtr(pMixedCtx);
7712	HMVMX_ASSERT_PREEMPT_SAFE();
7713
7714	#ifdef LOG_ENABLED
7715	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7716	* probably not initialized yet? Anyway this will do for now.
7717	*
7718	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7719	* interface and disable ring-3 calls when thread-context hooks are not
7720	* available. */
7721	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7722	VMMR0LogFlushDisable(pVCpu);
7723	#endif
7724
7725	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7726
7727	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7728
7729	/* Determine real-on-v86 mode. */
7730	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7731	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7732	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7733	{
7734	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7735	}
7736
7737	/*
7738	* Load the guest-state into the VMCS.
7739	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7740	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7741	*/
7742	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7743	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7744
7745	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7746	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7747	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7748
7749	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7750	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7751	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7752
7753	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7754	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7755
7756	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7757	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7758
7759	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7760	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7761	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7762
7763	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7764	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7765
7766	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7767	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7768
7769	/*
7770	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7771	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7772	*/
7773	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7774	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7775
7776	/* Clear any unused and reserved bits. */
7777	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7778
7779	#ifdef LOG_ENABLED
7780	/* Only reenable log-flushing if the caller has it enabled. */
7781	if (!fCallerDisabledLogFlush)
7782	VMMR0LogFlushEnable(pVCpu);
7783	#endif
7784
7785	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7786	return rc;
7787	}
7788
7789
7790	/**
7791	* Loads the state shared between the host and guest into the VMCS.
7792	*
7793	* @param pVM Pointer to the VM.
7794	* @param pVCpu Pointer to the VMCPU.
7795	* @param pCtx Pointer to the guest-CPU context.
7796	*
7797	* @remarks No-long-jump zone!!!
7798	*/
7799	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7800	{
7801	NOREF(pVM);
7802
7803	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7804	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7805
7806	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7807	{
7808	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7809	AssertRC(rc);
7810	}
7811
7812	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7813	{
7814	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7815	AssertRC(rc);
7816
7817	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7818	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7819	{
7820	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7821	AssertRC(rc);
7822	}
7823	}
7824
7825	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7826	{
7827	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7828	if ( HMVMX_IS_64BIT_HOST_MODE()
7829	&& pVM->hm.s.fAllow64BitGuests)
7830	{
7831	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7832	}
7833	#endif
7834	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7835	}
7836
7837	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7838	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7839	}
7840
7841
7842	/**
7843	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7844	*
7845	* @param pVM Pointer to the VM.
7846	* @param pVCpu Pointer to the VMCPU.
7847	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7848	* out-of-sync. Make sure to update the required fields
7849	* before using them.
7850	*/
7851	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7852	{
7853	HMVMX_ASSERT_PREEMPT_SAFE();
7854
7855	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7856	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7857	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7858	#endif
7859
7860	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7861	{
7862	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7863	AssertRC(rc);
7864	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7865	}
7866	else if (VMCPU_HMCF_VALUE(pVCpu))
7867	{
7868	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7869	AssertRC(rc);
7870	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7871	}
7872
7873	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7874	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7875	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7876	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7877
7878	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7879	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7880	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7881	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7882	#endif
7883	}
7884
7885
7886	/**
7887	* Does the preparations before executing guest code in VT-x.
7888	*
7889	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7890	* recompiler. We must be cautious what we do here regarding committing
7891	* guest-state information into the VMCS assuming we assuredly execute the
7892	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7893	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7894	* so that the recompiler can (and should) use them when it resumes guest
7895	* execution. Otherwise such operations must be done when we can no longer
7896	* exit to ring-3.
7897	*
7898	* @returns Strict VBox status code.
7899	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7900	* have been disabled.
7901	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7902	* double-fault into the guest.
7903	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7904	*
7905	* @param pVM Pointer to the VM.
7906	* @param pVCpu Pointer to the VMCPU.
7907	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7908	* out-of-sync. Make sure to update the required fields
7909	* before using them.
7910	* @param pVmxTransient Pointer to the VMX transient structure.
7911	*/
7912	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7913	{
7914	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7915
7916	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7917	PGMRZDynMapFlushAutoSet(pVCpu);
7918	#endif
7919
7920	/* Check force flag actions that might require us to go back to ring-3. */
7921	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7922	if (rc != VINF_SUCCESS)
7923	return rc;
7924
7925	#ifndef IEM_VERIFICATION_MODE_FULL
7926	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7927	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7928	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7929	{
7930	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7931	RTGCPHYS GCPhysApicBase;
7932	GCPhysApicBase = pMixedCtx->msrApicBase;
7933	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7934
7935	/* Unalias any existing mapping. */
7936	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7937	AssertRCReturn(rc, rc);
7938
7939	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7940	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7941	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7942	AssertRCReturn(rc, rc);
7943
7944	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7945	}
7946	#endif /* !IEM_VERIFICATION_MODE_FULL */
7947
7948	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7949	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7950
7951	/*
7952	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7953	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7954	*/
7955	if (TRPMHasTrap(pVCpu))
7956	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7957	else if (!pVCpu->hm.s.Event.fPending)
7958	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7959
7960	/*
7961	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7962	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7963	*/
7964	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7965	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7966	{
7967	Assert(rc == VINF_EM_RESET);
7968	return rc;
7969	}
7970
7971	/*
7972	* No longjmps to ring-3 from this point on!!!
7973	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7974	* This also disables flushing of the R0-logger instance (if any).
7975	*/
7976	VMMRZCallRing3Disable(pVCpu);
7977
7978	/*
7979	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7980	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7981	*
7982	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7983	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7984	*
7985	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7986	* executing guest code.
7987	*/
7988	pVmxTransient->uEflags = ASMIntDisableFlags();
7989	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7990	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7991	{
7992	hmR0VmxClearEventVmcs(pVCpu);
7993	ASMSetFlags(pVmxTransient->uEflags);
7994	VMMRZCallRing3Enable(pVCpu);
7995	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7996	return VINF_EM_RAW_TO_R3;
7997	}
7998	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7999	{
8000	hmR0VmxClearEventVmcs(pVCpu);
8001	ASMSetFlags(pVmxTransient->uEflags);
8002	VMMRZCallRing3Enable(pVCpu);
8003	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8004	return VINF_EM_RAW_INTERRUPT;
8005	}
8006
8007	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8008	pVCpu->hm.s.Event.fPending = false;
8009
8010	return VINF_SUCCESS;
8011	}
8012
8013
8014	/**
8015	* Prepares to run guest code in VT-x and we've committed to doing so. This
8016	* means there is no backing out to ring-3 or anywhere else at this
8017	* point.
8018	*
8019	* @param pVM Pointer to the VM.
8020	* @param pVCpu Pointer to the VMCPU.
8021	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8022	* out-of-sync. Make sure to update the required fields
8023	* before using them.
8024	* @param pVmxTransient Pointer to the VMX transient structure.
8025	*
8026	* @remarks Called with preemption disabled.
8027	* @remarks No-long-jump zone!!!
8028	*/
8029	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8030	{
8031	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8032	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8033	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8034
8035	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8036	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8037
8038	/*
8039	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8040	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8041	* Reload only the necessary state, the assertion will catch if other parts of the code
8042	* change.
8043	*/
8044	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8045	{
8046	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8047	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8048	}
8049
8050	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8051	if (!CPUMIsGuestFPUStateActive(pVCpu))
8052	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8053	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8054	#endif
8055
8056	if ( pVCpu->hm.s.fUseGuestFpu
8057	&& !CPUMIsGuestFPUStateActive(pVCpu))
8058	{
8059	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8060	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
8061	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8062	}
8063
8064	/*
8065	* The host MSR values the very first time around won't be updated, so we need to
8066	* fill those values in. Subsequently, it's updated as part of the host state.
8067	*/
8068	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8069	&& pVCpu->hm.s.vmx.cMsrs > 0)
8070	{
8071	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8072	}
8073
8074	/*
8075	* Load the host state bits as we may've been preempted (only happens when
8076	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8077	*/
8078	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8079	{
8080	/* This ASSUMES that pfnStartVM has been set up already. */
8081	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8082	AssertRC(rc);
8083	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8084	}
8085	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8086
8087	/*
8088	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8089	*/
8090	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8091	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8092	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
8093
8094	/* Store status of the shared guest-host state at the time of VM-entry. */
8095	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8096	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8097	{
8098	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8099	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8100	}
8101	else
8102	#endif
8103	{
8104	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8105	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8106	}
8107	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8108
8109	/*
8110	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8111	*/
8112	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8113	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8114
8115	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8116	RTCPUID idCurrentCpu = pCpu->idCpu;
8117	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8118	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8119	{
8120	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8121	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8122	}
8123
8124	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8125	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8126	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8127	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8128
8129	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8130
8131	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8132	to start executing. */
8133
8134	/*
8135	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8136	*/
8137	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8138	{
8139	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8140	{
8141	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8142	AssertRC(rc2);
8143	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
8144	uint64_t u64GuestTscAuxMsr;
8145	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTscAuxMsr);
8146	AssertRC(rc2);
8147	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, u64GuestTscAuxMsr, true /* fUpdateHostMsr */);
8148	}
8149	else
8150	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8151	}
8152	#ifdef VBOX_STRICT
8153	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8154	#endif
8155	}
8156
8157
8158	/**
8159	* Performs some essential restoration of state after running guest code in
8160	* VT-x.
8161	*
8162	* @param pVM Pointer to the VM.
8163	* @param pVCpu Pointer to the VMCPU.
8164	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8165	* out-of-sync. Make sure to update the required fields
8166	* before using them.
8167	* @param pVmxTransient Pointer to the VMX transient structure.
8168	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8169	*
8170	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8171	*
8172	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8173	* unconditionally when it is safe to do so.
8174	*/
8175	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8176	{
8177	NOREF(pVM);
8178
8179	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8180
8181	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8182	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8183	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8184	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8185	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8186
8187	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8188	{
8189	/** @todo Find a way to fix hardcoding a guestimate. */
8190	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8191	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8192	}
8193
8194	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8195	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8196	Assert(!(ASMGetFlags() & X86_EFL_IF));
8197	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8198
8199	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8200	if (CPUMIsGuestFPUStateActive(pVCpu))
8201	{
8202	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8203	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8204	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8205	}
8206	#endif
8207
8208	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8209	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8210	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8211	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8212
8213	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8214	uint32_t uExitReason;
8215	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8216	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8217	AssertRC(rc);
8218	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8219	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8220
8221	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8222	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8223	{
8224	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8225	pVmxTransient->fVMEntryFailed));
8226	return;
8227	}
8228
8229	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8230	{
8231	/* Update the guest interruptibility-state from the VMCS. */
8232	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8233	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8234	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8235	AssertRC(rc);
8236	#endif
8237	/*
8238	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8239	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8240	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8241	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8242	*/
8243	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8244	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8245	{
8246	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8247	AssertRC(rc);
8248	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8249	}
8250	}
8251	}
8252
8253
8254
8255	/**
8256	* Runs the guest code using VT-x the normal way.
8257	*
8258	* @returns VBox status code.
8259	* @param pVM Pointer to the VM.
8260	* @param pVCpu Pointer to the VMCPU.
8261	* @param pCtx Pointer to the guest-CPU context.
8262	*
8263	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8264	*/
8265	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8266	{
8267	VMXTRANSIENT VmxTransient;
8268	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8269	int rc = VERR_INTERNAL_ERROR_5;
8270	uint32_t cLoops = 0;
8271
8272	for (;; cLoops++)
8273	{
8274	Assert(!HMR0SuspendPending());
8275	HMVMX_ASSERT_CPU_SAFE();
8276
8277	/* Preparatory work for running guest code, this may force us to return
8278	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8279	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8280	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8281	if (rc != VINF_SUCCESS)
8282	break;
8283
8284	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8285	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8286	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8287
8288	/* Restore any residual host-state and save any bits shared between host
8289	and guest into the guest-CPU state. Re-enables interrupts! */
8290	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8291
8292	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8293	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8294	{
8295	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8296	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8297	return rc;
8298	}
8299
8300	/* Handle the VM-exit. */
8301	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8302	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8303	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8304	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8305	HMVMX_START_EXIT_DISPATCH_PROF();
8306	#ifdef HMVMX_USE_FUNCTION_TABLE
8307	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8308	#else
8309	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8310	#endif
8311	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8312	if (rc != VINF_SUCCESS)
8313	break;
8314	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8315	{
8316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8317	rc = VINF_EM_RAW_INTERRUPT;
8318	break;
8319	}
8320	}
8321
8322	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8323	return rc;
8324	}
8325
8326
8327	/**
8328	* Single steps guest code using VT-x.
8329	*
8330	* @returns VBox status code.
8331	* @param pVM Pointer to the VM.
8332	* @param pVCpu Pointer to the VMCPU.
8333	* @param pCtx Pointer to the guest-CPU context.
8334	*
8335	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8336	*/
8337	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8338	{
8339	VMXTRANSIENT VmxTransient;
8340	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8341	int rc = VERR_INTERNAL_ERROR_5;
8342	uint32_t cLoops = 0;
8343	uint16_t uCsStart = pCtx->cs.Sel;
8344	uint64_t uRipStart = pCtx->rip;
8345
8346	for (;; cLoops++)
8347	{
8348	Assert(!HMR0SuspendPending());
8349	HMVMX_ASSERT_CPU_SAFE();
8350
8351	/* Preparatory work for running guest code, this may force us to return
8352	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8353	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8354	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8355	if (rc != VINF_SUCCESS)
8356	break;
8357
8358	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8359	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8360	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8361
8362	/* Restore any residual host-state and save any bits shared between host
8363	and guest into the guest-CPU state. Re-enables interrupts! */
8364	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8365
8366	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8367	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8368	{
8369	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8370	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8371	return rc;
8372	}
8373
8374	/* Handle the VM-exit. */
8375	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8376	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8377	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8378	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8379	HMVMX_START_EXIT_DISPATCH_PROF();
8380	#ifdef HMVMX_USE_FUNCTION_TABLE
8381	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8382	#else
8383	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8384	#endif
8385	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8386	if (rc != VINF_SUCCESS)
8387	break;
8388	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8389	{
8390	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8391	rc = VINF_EM_RAW_INTERRUPT;
8392	break;
8393	}
8394
8395	/*
8396	* Did the RIP change, if so, consider it a single step.
8397	* Otherwise, make sure one of the TFs gets set.
8398	*/
8399	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8400	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8401	AssertRCReturn(rc2, rc2);
8402	if ( pCtx->rip != uRipStart
8403	\|\| pCtx->cs.Sel != uCsStart)
8404	{
8405	rc = VINF_EM_DBG_STEPPED;
8406	break;
8407	}
8408	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8409	}
8410
8411	/*
8412	* Clear the X86_EFL_TF if necessary.
8413	*/
8414	if (pVCpu->hm.s.fClearTrapFlag)
8415	{
8416	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8417	AssertRCReturn(rc2, rc2);
8418	pVCpu->hm.s.fClearTrapFlag = false;
8419	pCtx->eflags.Bits.u1TF = 0;
8420	}
8421	/** @todo there seems to be issues with the resume flag when the monitor trap
8422	* flag is pending without being used. Seen early in bios init when
8423	* accessing APIC page in prot mode. */
8424
8425	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8426	return rc;
8427	}
8428
8429
8430	/**
8431	* Runs the guest code using VT-x.
8432	*
8433	* @returns VBox status code.
8434	* @param pVM Pointer to the VM.
8435	* @param pVCpu Pointer to the VMCPU.
8436	* @param pCtx Pointer to the guest-CPU context.
8437	*/
8438	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8439	{
8440	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8441	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8442	HMVMX_ASSERT_PREEMPT_SAFE();
8443
8444	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8445
8446	int rc;
8447	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8448	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8449	else
8450	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8451
8452	if (rc == VERR_EM_INTERPRETER)
8453	rc = VINF_EM_RAW_EMULATE_INSTR;
8454	else if (rc == VINF_EM_RESET)
8455	rc = VINF_EM_TRIPLE_FAULT;
8456
8457	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8458	if (RT_FAILURE(rc2))
8459	{
8460	pVCpu->hm.s.u32HMError = rc;
8461	rc = rc2;
8462	}
8463	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8464	return rc;
8465	}
8466
8467
8468	#ifndef HMVMX_USE_FUNCTION_TABLE
8469	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8470	{
8471	#ifdef DEBUG_ramshankar
8472	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8473	# define LDVMCS() do { VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8474	#endif
8475	int rc;
8476	switch (rcReason)
8477	{
8478	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8479	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8480	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8481	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8482	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8483	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8484	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8485	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8486	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8487	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8488	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8489	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8490	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8491	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8492	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8493	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8494	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8495	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8496	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8497	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8498	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8499	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8500	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8508	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8509	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8510	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8511
8512	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8513	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8514	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8515	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8516	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8517	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8518	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8519	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8520	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8521
8522	case VMX_EXIT_VMCALL:
8523	case VMX_EXIT_VMCLEAR:
8524	case VMX_EXIT_VMLAUNCH:
8525	case VMX_EXIT_VMPTRLD:
8526	case VMX_EXIT_VMPTRST:
8527	case VMX_EXIT_VMREAD:
8528	case VMX_EXIT_VMRESUME:
8529	case VMX_EXIT_VMWRITE:
8530	case VMX_EXIT_VMXOFF:
8531	case VMX_EXIT_VMXON:
8532	case VMX_EXIT_INVEPT:
8533	case VMX_EXIT_INVVPID:
8534	case VMX_EXIT_VMFUNC:
8535	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8536	break;
8537	default:
8538	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8539	break;
8540	}
8541	return rc;
8542	}
8543	#endif
8544
8545	#ifdef DEBUG
8546	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8547	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8548	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8549
8550	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8551	do \
8552	{ \
8553	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8554	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8555	} while (0)
8556
8557	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8558	do { \
8559	AssertPtr(pVCpu); \
8560	AssertPtr(pMixedCtx); \
8561	AssertPtr(pVmxTransient); \
8562	Assert(pVmxTransient->fVMEntryFailed == false); \
8563	Assert(ASMIntAreEnabled()); \
8564	HMVMX_ASSERT_PREEMPT_SAFE(); \
8565	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8566	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8567	HMVMX_ASSERT_PREEMPT_SAFE(); \
8568	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8569	HMVMX_ASSERT_PREEMPT_CPUID(); \
8570	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8571	} while (0)
8572
8573	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8574	do { \
8575	Log4Func(("\n")); \
8576	} while (0)
8577	#else /* Release builds */
8578	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8579	do { \
8580	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8581	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8582	} while (0)
8583	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8584	#endif
8585
8586
8587	/**
8588	* Advances the guest RIP after reading it from the VMCS.
8589	*
8590	* @returns VBox status code.
8591	* @param pVCpu Pointer to the VMCPU.
8592	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8593	* out-of-sync. Make sure to update the required fields
8594	* before using them.
8595	* @param pVmxTransient Pointer to the VMX transient structure.
8596	*
8597	* @remarks No-long-jump zone!!!
8598	*/
8599	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8600	{
8601	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8602	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8603	AssertRCReturn(rc, rc);
8604
8605	pMixedCtx->rip += pVmxTransient->cbInstr;
8606	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8607	return rc;
8608	}
8609
8610
8611	/**
8612	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8613	* and update error record fields accordingly.
8614	*
8615	* @return VMX_IGS_* return codes.
8616	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8617	* wrong with the guest state.
8618	*
8619	* @param pVM Pointer to the VM.
8620	* @param pVCpu Pointer to the VMCPU.
8621	* @param pCtx Pointer to the guest-CPU state.
8622	*/
8623	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8624	{
8625	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8626	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8627	uError = (err); \
8628	break; \
8629	} else do { } while (0)
8630
8631	int rc;
8632	uint32_t uError = VMX_IGS_ERROR;
8633	uint32_t u32Val;
8634	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8635
8636	do
8637	{
8638	/*
8639	* CR0.
8640	*/
8641	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8642	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8643	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8644	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8645	if (fUnrestrictedGuest)
8646	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8647
8648	uint32_t u32GuestCR0;
8649	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8650	AssertRCBreak(rc);
8651	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8652	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8653	if ( !fUnrestrictedGuest
8654	&& (u32GuestCR0 & X86_CR0_PG)
8655	&& !(u32GuestCR0 & X86_CR0_PE))
8656	{
8657	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8658	}
8659
8660	/*
8661	* CR4.
8662	*/
8663	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8664	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8665
8666	uint32_t u32GuestCR4;
8667	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8668	AssertRCBreak(rc);
8669	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8670	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8671
8672	/*
8673	* IA32_DEBUGCTL MSR.
8674	*/
8675	uint64_t u64Val;
8676	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8677	AssertRCBreak(rc);
8678	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8679	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8680	{
8681	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8682	}
8683	uint64_t u64DebugCtlMsr = u64Val;
8684
8685	#ifdef VBOX_STRICT
8686	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8687	AssertRCBreak(rc);
8688	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8689	#endif
8690	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8691
8692	/*
8693	* RIP and RFLAGS.
8694	*/
8695	uint32_t u32Eflags;
8696	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8697	if (HMVMX_IS_64BIT_HOST_MODE())
8698	{
8699	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8700	AssertRCBreak(rc);
8701	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8702	if ( !fLongModeGuest
8703	\|\| !pCtx->cs.Attr.n.u1Long)
8704	{
8705	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8706	}
8707	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8708	* must be identical if the "IA32e mode guest" VM-entry control is 1
8709	* and CS.L is 1. No check applies if the CPU supports 64
8710	* linear-address bits. */
8711
8712	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8713	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8714	AssertRCBreak(rc);
8715	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8716	VMX_IGS_RFLAGS_RESERVED);
8717	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8718	u32Eflags = u64Val;
8719	}
8720	else
8721	#endif
8722	{
8723	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8724	AssertRCBreak(rc);
8725	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8726	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8727	}
8728
8729	if ( fLongModeGuest
8730	\|\| ( fUnrestrictedGuest
8731	&& !(u32GuestCR0 & X86_CR0_PE)))
8732	{
8733	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8734	}
8735
8736	uint32_t u32EntryInfo;
8737	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8738	AssertRCBreak(rc);
8739	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8740	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8741	{
8742	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8743	}
8744
8745	/*
8746	* 64-bit checks.
8747	*/
8748	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8749	if (HMVMX_IS_64BIT_HOST_MODE())
8750	{
8751	if ( fLongModeGuest
8752	&& !fUnrestrictedGuest)
8753	{
8754	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8755	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8756	}
8757
8758	if ( !fLongModeGuest
8759	&& (u32GuestCR4 & X86_CR4_PCIDE))
8760	{
8761	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8762	}
8763
8764	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8765	* 51:32 beyond the processor's physical-address width are 0. */
8766
8767	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8768	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8769	{
8770	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8771	}
8772
8773	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8774	AssertRCBreak(rc);
8775	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8776
8777	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8778	AssertRCBreak(rc);
8779	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8780	}
8781	#endif
8782
8783	/*
8784	* PERF_GLOBAL MSR.
8785	*/
8786	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8787	{
8788	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8789	AssertRCBreak(rc);
8790	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8791	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8792	}
8793
8794	/*
8795	* PAT MSR.
8796	*/
8797	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8798	{
8799	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8800	AssertRCBreak(rc);
8801	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8802	for (unsigned i = 0; i < 8; i++)
8803	{
8804	uint8_t u8Val = (u64Val & 0x7);
8805	if ( u8Val != 0 /* UC */
8806	\|\| u8Val != 1 /* WC */
8807	\|\| u8Val != 4 /* WT */
8808	\|\| u8Val != 5 /* WP */
8809	\|\| u8Val != 6 /* WB */
8810	\|\| u8Val != 7 /* UC- */)
8811	{
8812	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8813	}
8814	u64Val >>= 3;
8815	}
8816	}
8817
8818	/*
8819	* EFER MSR.
8820	*/
8821	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8822	{
8823	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8824	AssertRCBreak(rc);
8825	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8826	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8827	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8828	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8829	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8830	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8831	}
8832
8833	/*
8834	* Segment registers.
8835	*/
8836	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8837	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8838	if (!(u32Eflags & X86_EFL_VM))
8839	{
8840	/* CS */
8841	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8842	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8843	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8844	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8845	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8846	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8847	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8848	/* CS cannot be loaded with NULL in protected mode. */
8849	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8850	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8851	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8852	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8853	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8854	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8855	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8856	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8857	else
8858	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8859
8860	/* SS */
8861	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8862	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8863	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8864	if ( !(pCtx->cr0 & X86_CR0_PE)
8865	\|\| pCtx->cs.Attr.n.u4Type == 3)
8866	{
8867	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8868	}
8869	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8870	{
8871	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8872	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8873	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8874	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8875	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8876	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8877	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8878	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8879	}
8880
8881	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8882	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8883	{
8884	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8885	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8886	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8887	\|\| pCtx->ds.Attr.n.u4Type > 11
8888	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8889	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8890	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8891	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8892	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8893	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8894	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8895	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8896	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8897	}
8898	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8899	{
8900	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8901	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8902	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8903	\|\| pCtx->es.Attr.n.u4Type > 11
8904	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8905	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8906	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8907	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8908	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8909	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8910	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8911	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8912	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8913	}
8914	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8915	{
8916	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8917	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8918	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8919	\|\| pCtx->fs.Attr.n.u4Type > 11
8920	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8921	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8922	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8923	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8924	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8925	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8926	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8927	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8928	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8929	}
8930	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8931	{
8932	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8933	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8934	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8935	\|\| pCtx->gs.Attr.n.u4Type > 11
8936	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8937	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8938	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8939	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8940	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8941	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8942	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8943	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8944	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8945	}
8946	/* 64-bit capable CPUs. */
8947	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8948	if (HMVMX_IS_64BIT_HOST_MODE())
8949	{
8950	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8951	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8952	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8953	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8954	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8955	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8956	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8957	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8958	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8959	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8960	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8961	}
8962	#endif
8963	}
8964	else
8965	{
8966	/* V86 mode checks. */
8967	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8968	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8969	{
8970	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8971	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8972	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8973	}
8974	else
8975	{
8976	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8977	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8978	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8979	}
8980
8981	/* CS */
8982	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8983	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8984	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8985	/* SS */
8986	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8987	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8988	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8989	/* DS */
8990	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8991	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8992	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8993	/* ES */
8994	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8995	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8996	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8997	/* FS */
8998	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8999	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9000	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9001	/* GS */
9002	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9003	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9004	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9005	/* 64-bit capable CPUs. */
9006	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9007	if (HMVMX_IS_64BIT_HOST_MODE())
9008	{
9009	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9010	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9011	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9012	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9013	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9014	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9015	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9016	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9017	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9018	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9019	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9020	}
9021	#endif
9022	}
9023
9024	/*
9025	* TR.
9026	*/
9027	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9028	/* 64-bit capable CPUs. */
9029	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9030	if (HMVMX_IS_64BIT_HOST_MODE())
9031	{
9032	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9033	}
9034	#endif
9035	if (fLongModeGuest)
9036	{
9037	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9038	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9039	}
9040	else
9041	{
9042	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9043	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9044	VMX_IGS_TR_ATTR_TYPE_INVALID);
9045	}
9046	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9047	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9048	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9049	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9050	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9051	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9052	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9053	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9054
9055	/*
9056	* GDTR and IDTR.
9057	*/
9058	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9059	if (HMVMX_IS_64BIT_HOST_MODE())
9060	{
9061	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9062	AssertRCBreak(rc);
9063	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9064
9065	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9066	AssertRCBreak(rc);
9067	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9068	}
9069	#endif
9070
9071	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9072	AssertRCBreak(rc);
9073	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9074
9075	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9076	AssertRCBreak(rc);
9077	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9078
9079	/*
9080	* Guest Non-Register State.
9081	*/
9082	/* Activity State. */
9083	uint32_t u32ActivityState;
9084	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9085	AssertRCBreak(rc);
9086	HMVMX_CHECK_BREAK( !u32ActivityState
9087	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9088	VMX_IGS_ACTIVITY_STATE_INVALID);
9089	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9090	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9091	uint32_t u32IntrState;
9092	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9093	AssertRCBreak(rc);
9094	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9095	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9096	{
9097	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9098	}
9099
9100	/** @todo Activity state and injecting interrupts. Left as a todo since we
9101	* currently don't use activity states but ACTIVE. */
9102
9103	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9104	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9105
9106	/* Guest interruptibility-state. */
9107	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9108	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9109	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9110	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9111	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9112	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9113	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9114	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9115	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9116	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9117	{
9118	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9119	{
9120	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9121	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9122	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9123	}
9124	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9125	{
9126	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9127	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9128	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9129	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9130	}
9131	}
9132	/** @todo Assumes the processor is not in SMM. */
9133	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9134	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9135	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9136	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9137	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9138	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9139	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9140	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9141	{
9142	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9143	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9144	}
9145
9146	/* Pending debug exceptions. */
9147	if (HMVMX_IS_64BIT_HOST_MODE())
9148	{
9149	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9150	AssertRCBreak(rc);
9151	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9152	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9153	u32Val = u64Val; /* For pending debug exceptions checks below. */
9154	}
9155	else
9156	{
9157	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9158	AssertRCBreak(rc);
9159	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9160	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9161	}
9162
9163	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9164	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9165	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9166	{
9167	if ( (u32Eflags & X86_EFL_TF)
9168	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9169	{
9170	/* Bit 14 is PendingDebug.BS. */
9171	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9172	}
9173	if ( !(u32Eflags & X86_EFL_TF)
9174	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9175	{
9176	/* Bit 14 is PendingDebug.BS. */
9177	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9178	}
9179	}
9180
9181	/* VMCS link pointer. */
9182	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9183	AssertRCBreak(rc);
9184	if (u64Val != UINT64_C(0xffffffffffffffff))
9185	{
9186	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9187	/** @todo Bits beyond the processor's physical-address width MBZ. */
9188	/** @todo 32-bit located in memory referenced by value of this field (as a
9189	* physical address) must contain the processor's VMCS revision ID. */
9190	/** @todo SMM checks. */
9191	}
9192
9193	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
9194
9195	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9196	if (uError == VMX_IGS_ERROR)
9197	uError = VMX_IGS_REASON_NOT_FOUND;
9198	} while (0);
9199
9200	pVCpu->hm.s.u32HMError = uError;
9201	return uError;
9202
9203	#undef HMVMX_ERROR_BREAK
9204	#undef HMVMX_CHECK_BREAK
9205	}
9206
9207	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9208	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9209	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9210
9211	/** @name VM-exit handlers.
9212	* @{
9213	*/
9214
9215	/**
9216	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9217	*/
9218	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9219	{
9220	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9221	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9222	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9223	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9224	return VINF_SUCCESS;
9225	return VINF_EM_RAW_INTERRUPT;
9226	}
9227
9228
9229	/**
9230	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9231	*/
9232	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9233	{
9234	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9235	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9236
9237	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9238	AssertRCReturn(rc, rc);
9239
9240	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9241	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9242	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9243	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9244
9245	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9246	{
9247	/*
9248	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9249	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9250	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9251	*
9252	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9253	*/
9254	VMXDispatchHostNmi();
9255	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9256	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9257	return VINF_SUCCESS;
9258	}
9259
9260	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9261	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9262	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9263	{
9264	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9265	return VINF_SUCCESS;
9266	}
9267	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9268	{
9269	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9270	return rc;
9271	}
9272
9273	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9274	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9275	switch (uIntType)
9276	{
9277	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9278	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9279	/* no break */
9280	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9281	{
9282	switch (uVector)
9283	{
9284	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9285	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9286	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9287	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9288	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9289	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9290	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9291	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9292	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9293	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9294	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9295	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9296	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9297	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9298	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9299	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9300	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9301	#endif
9302	default:
9303	{
9304	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9305	AssertRCReturn(rc, rc);
9306
9307	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9308	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9309	{
9310	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9311	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9312	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9313
9314	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9315	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9316	AssertRCReturn(rc, rc);
9317	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9318	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9319	0 /* GCPtrFaultAddress */);
9320	AssertRCReturn(rc, rc);
9321	}
9322	else
9323	{
9324	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9325	pVCpu->hm.s.u32HMError = uVector;
9326	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9327	}
9328	break;
9329	}
9330	}
9331	break;
9332	}
9333
9334	default:
9335	{
9336	pVCpu->hm.s.u32HMError = uExitIntInfo;
9337	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9338	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9339	break;
9340	}
9341	}
9342	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9343	return rc;
9344	}
9345
9346
9347	/**
9348	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9349	*/
9350	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9351	{
9352	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9353
9354	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9355	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9356	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9357	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9358	AssertRCReturn(rc, rc);
9359
9360	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9361	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9362	return VINF_SUCCESS;
9363	}
9364
9365
9366	/**
9367	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9368	*/
9369	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9370	{
9371	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9372	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9373	HMVMX_RETURN_UNEXPECTED_EXIT();
9374	}
9375
9376
9377	/**
9378	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9379	*/
9380	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9381	{
9382	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9383	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9384	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9385	}
9386
9387
9388	/**
9389	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9390	*/
9391	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9392	{
9393	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9394	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9395	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9396	}
9397
9398
9399	/**
9400	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9401	*/
9402	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9403	{
9404	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9405	PVM pVM = pVCpu->CTX_SUFF(pVM);
9406	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9407	if (RT_LIKELY(rc == VINF_SUCCESS))
9408	{
9409	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9410	Assert(pVmxTransient->cbInstr == 2);
9411	}
9412	else
9413	{
9414	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9415	rc = VERR_EM_INTERPRETER;
9416	}
9417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9418	return rc;
9419	}
9420
9421
9422	/**
9423	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9424	*/
9425	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9426	{
9427	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9428	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9429	AssertRCReturn(rc, rc);
9430
9431	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9432	return VINF_EM_RAW_EMULATE_INSTR;
9433
9434	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9435	HMVMX_RETURN_UNEXPECTED_EXIT();
9436	}
9437
9438
9439	/**
9440	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9441	*/
9442	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9443	{
9444	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9445	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9446	AssertRCReturn(rc, rc);
9447
9448	PVM pVM = pVCpu->CTX_SUFF(pVM);
9449	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9450	if (RT_LIKELY(rc == VINF_SUCCESS))
9451	{
9452	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9453	Assert(pVmxTransient->cbInstr == 2);
9454	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9455	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9456	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9457	}
9458	else
9459	{
9460	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9461	rc = VERR_EM_INTERPRETER;
9462	}
9463	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9464	return rc;
9465	}
9466
9467
9468	/**
9469	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9470	*/
9471	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9472	{
9473	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9474	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9475	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9476	AssertRCReturn(rc, rc);
9477
9478	PVM pVM = pVCpu->CTX_SUFF(pVM);
9479	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9480	if (RT_LIKELY(rc == VINF_SUCCESS))
9481	{
9482	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9483	Assert(pVmxTransient->cbInstr == 3);
9484	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9485	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9486	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9487	}
9488	else
9489	{
9490	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9491	rc = VERR_EM_INTERPRETER;
9492	}
9493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9494	return rc;
9495	}
9496
9497
9498	/**
9499	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9500	*/
9501	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9502	{
9503	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9504	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9505	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9506	AssertRCReturn(rc, rc);
9507
9508	PVM pVM = pVCpu->CTX_SUFF(pVM);
9509	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9510	if (RT_LIKELY(rc == VINF_SUCCESS))
9511	{
9512	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9513	Assert(pVmxTransient->cbInstr == 2);
9514	}
9515	else
9516	{
9517	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9518	rc = VERR_EM_INTERPRETER;
9519	}
9520	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9521	return rc;
9522	}
9523
9524
9525	/**
9526	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9527	*/
9528	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9529	{
9530	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9531	PVM pVM = pVCpu->CTX_SUFF(pVM);
9532	Assert(!pVM->hm.s.fNestedPaging);
9533
9534	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9535	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9536	AssertRCReturn(rc, rc);
9537
9538	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9539	rc = VBOXSTRICTRC_VAL(rc2);
9540	if (RT_LIKELY(rc == VINF_SUCCESS))
9541	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9542	else
9543	{
9544	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9545	pVmxTransient->uExitQualification, rc));
9546	}
9547	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9548	return rc;
9549	}
9550
9551
9552	/**
9553	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9554	*/
9555	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9556	{
9557	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9558	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9559	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9560	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9561	AssertRCReturn(rc, rc);
9562
9563	PVM pVM = pVCpu->CTX_SUFF(pVM);
9564	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9565	if (RT_LIKELY(rc == VINF_SUCCESS))
9566	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9567	else
9568	{
9569	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9570	rc = VERR_EM_INTERPRETER;
9571	}
9572	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9573	return rc;
9574	}
9575
9576
9577	/**
9578	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9579	*/
9580	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9581	{
9582	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9583	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9584	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9585	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9586	AssertRCReturn(rc, rc);
9587
9588	PVM pVM = pVCpu->CTX_SUFF(pVM);
9589	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9590	rc = VBOXSTRICTRC_VAL(rc2);
9591	if (RT_LIKELY( rc == VINF_SUCCESS
9592	\|\| rc == VINF_EM_HALT))
9593	{
9594	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9595	AssertRCReturn(rc3, rc3);
9596
9597	if ( rc == VINF_EM_HALT
9598	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9599	{
9600	rc = VINF_SUCCESS;
9601	}
9602	}
9603	else
9604	{
9605	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9606	rc = VERR_EM_INTERPRETER;
9607	}
9608	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9609	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9611	return rc;
9612	}
9613
9614
9615	/**
9616	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9617	*/
9618	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9619	{
9620	/*
9621	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9622	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9623	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9624	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9625	*/
9626	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9627	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9628	HMVMX_RETURN_UNEXPECTED_EXIT();
9629	}
9630
9631
9632	/**
9633	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9634	*/
9635	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9636	{
9637	/*
9638	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9639	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9640	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9641	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9642	*/
9643	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9644	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9645	HMVMX_RETURN_UNEXPECTED_EXIT();
9646	}
9647
9648
9649	/**
9650	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9651	*/
9652	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9653	{
9654	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9655	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9656	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9657	HMVMX_RETURN_UNEXPECTED_EXIT();
9658	}
9659
9660
9661	/**
9662	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9663	*/
9664	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9665	{
9666	/*
9667	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9668	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9669	* See Intel spec. 25.3 "Other Causes of VM-exits".
9670	*/
9671	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9672	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9673	HMVMX_RETURN_UNEXPECTED_EXIT();
9674	}
9675
9676
9677	/**
9678	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9679	* VM-exit.
9680	*/
9681	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9682	{
9683	/*
9684	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9685	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9686	*
9687	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9688	* See Intel spec. "23.8 Restrictions on VMX operation".
9689	*/
9690	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9691	return VINF_SUCCESS;
9692	}
9693
9694
9695	/**
9696	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9697	* VM-exit.
9698	*/
9699	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9700	{
9701	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9702	return VINF_EM_RESET;
9703	}
9704
9705
9706	/**
9707	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9708	*/
9709	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9710	{
9711	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9712	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9713	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9714	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9715	AssertRCReturn(rc, rc);
9716
9717	pMixedCtx->rip++;
9718	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9719	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9720	rc = VINF_SUCCESS;
9721	else
9722	rc = VINF_EM_HALT;
9723
9724	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9725	return rc;
9726	}
9727
9728
9729	/**
9730	* VM-exit handler for instructions that result in a #UD exception delivered to
9731	* the guest.
9732	*/
9733	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9734	{
9735	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9736	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9737	return VINF_SUCCESS;
9738	}
9739
9740
9741	/**
9742	* VM-exit handler for expiry of the VMX preemption timer.
9743	*/
9744	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9745	{
9746	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9747
9748	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9749	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9750
9751	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9752	PVM pVM = pVCpu->CTX_SUFF(pVM);
9753	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9754	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9755	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9756	}
9757
9758
9759	/**
9760	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9761	*/
9762	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9763	{
9764	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9765
9766	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9767	/** @todo check if XSETBV is supported by the recompiler. */
9768	return VERR_EM_INTERPRETER;
9769	}
9770
9771
9772	/**
9773	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9774	*/
9775	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9776	{
9777	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9778
9779	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9780	/** @todo implement EMInterpretInvpcid() */
9781	return VERR_EM_INTERPRETER;
9782	}
9783
9784
9785	/**
9786	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9787	* Error VM-exit.
9788	*/
9789	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9790	{
9791	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9792	AssertRCReturn(rc, rc);
9793
9794	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9795	NOREF(uInvalidReason);
9796
9797	#ifdef VBOX_STRICT
9798	uint32_t uIntrState;
9799	HMVMXHCUINTREG uHCReg;
9800	uint64_t u64Val;
9801	uint32_t u32Val;
9802
9803	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9804	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9805	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9806	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9807	AssertRCReturn(rc, rc);
9808
9809	Log4(("uInvalidReason %u\n", uInvalidReason));
9810	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9811	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9812	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9813	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9814
9815	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9816	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9817	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9818	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9819	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9820	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9821	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9822	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9823	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9824	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9825	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9826	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9827	#else
9828	NOREF(pVmxTransient);
9829	#endif
9830
9831	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9832	return VERR_VMX_INVALID_GUEST_STATE;
9833	}
9834
9835
9836	/**
9837	* VM-exit handler for VM-entry failure due to an MSR-load
9838	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9839	*/
9840	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9841	{
9842	NOREF(pVmxTransient);
9843	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9844	HMVMX_RETURN_UNEXPECTED_EXIT();
9845	}
9846
9847
9848	/**
9849	* VM-exit handler for VM-entry failure due to a machine-check event
9850	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9851	*/
9852	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9853	{
9854	NOREF(pVmxTransient);
9855	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9856	HMVMX_RETURN_UNEXPECTED_EXIT();
9857	}
9858
9859
9860	/**
9861	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9862	* theory.
9863	*/
9864	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9865	{
9866	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9867	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9868	return VERR_VMX_UNDEFINED_EXIT_CODE;
9869	}
9870
9871
9872	/**
9873	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9874	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9875	* Conditional VM-exit.
9876	*/
9877	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9878	{
9879	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9880
9881	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9882	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9883	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9884	return VERR_EM_INTERPRETER;
9885	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9886	HMVMX_RETURN_UNEXPECTED_EXIT();
9887	}
9888
9889
9890	/**
9891	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9892	*/
9893	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9894	{
9895	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9896
9897	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9899	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9900	return VERR_EM_INTERPRETER;
9901	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9902	HMVMX_RETURN_UNEXPECTED_EXIT();
9903	}
9904
9905
9906	/**
9907	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9908	*/
9909	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9910	{
9911	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9912
9913	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9914	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9915	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9916	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9917	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9918	{
9919	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9920	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9921	}
9922	AssertRCReturn(rc, rc);
9923	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9924
9925	#ifdef VBOX_STRICT
9926	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9927	{
9928	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9929	{
9930	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9931	HMVMX_RETURN_UNEXPECTED_EXIT();
9932	}
9933	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9934	if ( HMVMX_IS_64BIT_HOST_MODE()
9935	&& pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
9936	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
9937	{
9938	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
9939	HMVMX_RETURN_UNEXPECTED_EXIT();
9940	}
9941	# endif
9942	}
9943	#endif
9944
9945	PVM pVM = pVCpu->CTX_SUFF(pVM);
9946	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9947	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9948	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9950
9951	if (RT_LIKELY(rc == VINF_SUCCESS))
9952	{
9953	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9954	Assert(pVmxTransient->cbInstr == 2);
9955	}
9956	return rc;
9957	}
9958
9959
9960	/**
9961	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9962	*/
9963	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9964	{
9965	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9966	PVM pVM = pVCpu->CTX_SUFF(pVM);
9967	int rc = VINF_SUCCESS;
9968
9969	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9970	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9971	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9972	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9973	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9974	{
9975	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9976	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9977	}
9978	AssertRCReturn(rc, rc);
9979	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9980
9981	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9982	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9984
9985	if (RT_LIKELY(rc == VINF_SUCCESS))
9986	{
9987	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9988
9989	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9990	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9991	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9992	{
9993	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9994	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9995	EMInterpretWrmsr() changes it. */
9996	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9997	}
9998	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9999	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10000
10001	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10002	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10003	{
10004	switch (pMixedCtx->ecx)
10005	{
10006	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10007	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10008	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10009	case MSR_K8_FS_BASE: /* no break */
10010	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10011	default:
10012	{
10013	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10014	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10015	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10016	else if ( HMVMX_IS_64BIT_HOST_MODE()
10017	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10018	{
10019	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10020	}
10021	#endif
10022	break;
10023	}
10024	}
10025	}
10026	#ifdef VBOX_STRICT
10027	else
10028	{
10029	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10030	switch (pMixedCtx->ecx)
10031	{
10032	case MSR_IA32_SYSENTER_CS:
10033	case MSR_IA32_SYSENTER_EIP:
10034	case MSR_IA32_SYSENTER_ESP:
10035	case MSR_K8_FS_BASE:
10036	case MSR_K8_GS_BASE:
10037	{
10038	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10039	HMVMX_RETURN_UNEXPECTED_EXIT();
10040	}
10041
10042	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10043	default:
10044	{
10045	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10046	{
10047	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10048	pMixedCtx->ecx));
10049	HMVMX_RETURN_UNEXPECTED_EXIT();
10050	}
10051
10052	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10053	if ( HMVMX_IS_64BIT_HOST_MODE()
10054	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10055	{
10056	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10057	HMVMX_RETURN_UNEXPECTED_EXIT();
10058	}
10059	#endif
10060	break;
10061	}
10062	}
10063	}
10064	#endif /* VBOX_STRICT */
10065	}
10066	return rc;
10067	}
10068
10069
10070	/**
10071	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10072	*/
10073	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10074	{
10075	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10076
10077	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10078	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10079	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10080	return VERR_EM_INTERPRETER;
10081	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10082	HMVMX_RETURN_UNEXPECTED_EXIT();
10083	}
10084
10085
10086	/**
10087	* VM-exit handler for when the TPR value is lowered below the specified
10088	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10089	*/
10090	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10091	{
10092	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10093	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10094
10095	/*
10096	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10097	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10098	* resume guest execution.
10099	*/
10100	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10101	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10102	return VINF_SUCCESS;
10103	}
10104
10105
10106	/**
10107	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10108	* VM-exit.
10109	*
10110	* @retval VINF_SUCCESS when guest execution can continue.
10111	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10112	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10113	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10114	* recompiler.
10115	*/
10116	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10117	{
10118	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10119	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10120	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10121	AssertRCReturn(rc, rc);
10122
10123	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10124	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10125	PVM pVM = pVCpu->CTX_SUFF(pVM);
10126	switch (uAccessType)
10127	{
10128	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10129	{
10130	#if 0
10131	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10132	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10133	#else
10134	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10135	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10136	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10137	#endif
10138	AssertRCReturn(rc, rc);
10139
10140	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10141	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10142	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10143	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10144
10145	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10146	{
10147	case 0: /* CR0 */
10148	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10149	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10150	break;
10151	case 2: /* CR2 */
10152	/* Nothing to do here, CR2 it's not part of the VMCS. */
10153	break;
10154	case 3: /* CR3 */
10155	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10156	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10157	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10158	break;
10159	case 4: /* CR4 */
10160	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10161	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10162	break;
10163	case 8: /* CR8 */
10164	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10165	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10166	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10167	break;
10168	default:
10169	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10170	break;
10171	}
10172
10173	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10174	break;
10175	}
10176
10177	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10178	{
10179	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10180	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10181	AssertRCReturn(rc, rc);
10182	Assert( !pVM->hm.s.fNestedPaging
10183	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10184	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10185
10186	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10187	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10188	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10189
10190	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10191	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10192	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10193	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10194	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10195	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10196	break;
10197	}
10198
10199	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10200	{
10201	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10202	AssertRCReturn(rc, rc);
10203	rc = EMInterpretCLTS(pVM, pVCpu);
10204	AssertRCReturn(rc, rc);
10205	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10206	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10207	Log4(("CRX CLTS write rc=%d\n", rc));
10208	break;
10209	}
10210
10211	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10212	{
10213	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10214	AssertRCReturn(rc, rc);
10215	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10216	if (RT_LIKELY(rc == VINF_SUCCESS))
10217	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10218	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10219	Log4(("CRX LMSW write rc=%d\n", rc));
10220	break;
10221	}
10222
10223	default:
10224	{
10225	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10226	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10227	}
10228	}
10229
10230	/* Validate possible error codes. */
10231	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10232	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10233	if (RT_SUCCESS(rc))
10234	{
10235	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10236	AssertRCReturn(rc2, rc2);
10237	}
10238
10239	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10240	return rc;
10241	}
10242
10243
10244	/**
10245	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10246	* VM-exit.
10247	*/
10248	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10249	{
10250	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10251	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10252
10253	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10254	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10255	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10256	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10257	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10258	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10259	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10260	AssertRCReturn(rc2, rc2);
10261
10262	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10263	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10264	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10265	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10266	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10267	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10268	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10269
10270	/* I/O operation lookup arrays. */
10271	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10272	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10273
10274	VBOXSTRICTRC rcStrict;
10275	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10276	const uint32_t cbInstr = pVmxTransient->cbInstr;
10277	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10278	PVM pVM = pVCpu->CTX_SUFF(pVM);
10279	if (fIOString)
10280	{
10281	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10282	/*
10283	* INS/OUTS - I/O String instruction.
10284	*
10285	* Use instruction-information if available, otherwise fall back on
10286	* interpreting the instruction.
10287	*/
10288	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10289	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10290	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10291	{
10292	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10293	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10294	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10295	AssertRCReturn(rc2, rc2);
10296	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10297	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10298	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10299	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10300	if (fIOWrite)
10301	{
10302	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10303	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10304	}
10305	else
10306	{
10307	/*
10308	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10309	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10310	* See Intel Instruction spec. for "INS".
10311	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10312	*/
10313	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10314	}
10315	}
10316	else
10317	{
10318	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10319	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10320	AssertRCReturn(rc2, rc2);
10321	rcStrict = IEMExecOne(pVCpu);
10322	}
10323	/** @todo IEM needs to be setting these flags somehow. */
10324	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10325	fUpdateRipAlready = true;
10326	#else
10327	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10328	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10329	if (RT_SUCCESS(rcStrict))
10330	{
10331	if (fIOWrite)
10332	{
10333	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10334	(DISCPUMODE)pDis->uAddrMode, cbValue);
10335	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10336	}
10337	else
10338	{
10339	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10340	(DISCPUMODE)pDis->uAddrMode, cbValue);
10341	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10342	}
10343	}
10344	else
10345	{
10346	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10347	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10348	}
10349	#endif
10350	}
10351	else
10352	{
10353	/*
10354	* IN/OUT - I/O instruction.
10355	*/
10356	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10357	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10358	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10359	if (fIOWrite)
10360	{
10361	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10362	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10363	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10364	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10365	}
10366	else
10367	{
10368	uint32_t u32Result = 0;
10369	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10370	if (IOM_SUCCESS(rcStrict))
10371	{
10372	/* Save result of I/O IN instr. in AL/AX/EAX. */
10373	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10374	}
10375	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10376	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10377	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10378	}
10379	}
10380
10381	if (IOM_SUCCESS(rcStrict))
10382	{
10383	if (!fUpdateRipAlready)
10384	{
10385	pMixedCtx->rip += cbInstr;
10386	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10387	}
10388
10389	/*
10390	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10391	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10392	*/
10393	if (fIOString)
10394	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10395
10396	/*
10397	* If any I/O breakpoints are armed, we need to check if one triggered
10398	* and take appropriate action.
10399	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10400	*/
10401	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10402	AssertRCReturn(rc2, rc2);
10403
10404	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10405	* execution engines about whether hyper BPs and such are pending. */
10406	uint32_t const uDr7 = pMixedCtx->dr[7];
10407	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10408	&& X86_DR7_ANY_RW_IO(uDr7)
10409	&& (pMixedCtx->cr4 & X86_CR4_DE))
10410	\|\| DBGFBpIsHwIoArmed(pVM)))
10411	{
10412	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10413
10414	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10415	VMMRZCallRing3Disable(pVCpu);
10416	HM_DISABLE_PREEMPT_IF_NEEDED();
10417
10418	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10419
10420	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10421	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10422	{
10423	/* Raise #DB. */
10424	if (fIsGuestDbgActive)
10425	ASMSetDR6(pMixedCtx->dr[6]);
10426	if (pMixedCtx->dr[7] != uDr7)
10427	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10428
10429	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10430	}
10431	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10432	else if ( rcStrict2 != VINF_SUCCESS
10433	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10434	rcStrict = rcStrict2;
10435
10436	HM_RESTORE_PREEMPT_IF_NEEDED();
10437	VMMRZCallRing3Enable(pVCpu);
10438	}
10439	}
10440
10441	#ifdef DEBUG
10442	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10443	Assert(!fIOWrite);
10444	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10445	Assert(fIOWrite);
10446	else
10447	{
10448	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10449	* statuses, that the VMM device and some others may return. See
10450	* IOM_SUCCESS() for guidance. */
10451	AssertMsg( RT_FAILURE(rcStrict)
10452	\|\| rcStrict == VINF_SUCCESS
10453	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10454	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10455	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10456	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10457	}
10458	#endif
10459
10460	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10461	return VBOXSTRICTRC_TODO(rcStrict);
10462	}
10463
10464
10465	/**
10466	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10467	* VM-exit.
10468	*/
10469	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10470	{
10471	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10472
10473	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10474	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10475	AssertRCReturn(rc, rc);
10476	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10477	{
10478	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10479	AssertRCReturn(rc, rc);
10480	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10481	{
10482	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10483
10484	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10485	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10486	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10487	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10488	{
10489	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10490	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10491
10492	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10493	Assert(!pVCpu->hm.s.Event.fPending);
10494	pVCpu->hm.s.Event.fPending = true;
10495	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10496	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10497	AssertRCReturn(rc, rc);
10498	if (fErrorCodeValid)
10499	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10500	else
10501	pVCpu->hm.s.Event.u32ErrCode = 0;
10502	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10503	&& uVector == X86_XCPT_PF)
10504	{
10505	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10506	}
10507
10508	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10509	}
10510	}
10511	}
10512
10513	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10514	* emulation. */
10515	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10516	return VERR_EM_INTERPRETER;
10517	}
10518
10519
10520	/**
10521	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10522	*/
10523	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10524	{
10525	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10526	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10527	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10528	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10529	AssertRCReturn(rc, rc);
10530	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10531	return VINF_EM_DBG_STEPPED;
10532	}
10533
10534
10535	/**
10536	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10537	*/
10538	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10539	{
10540	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10541
10542	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10543	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10544	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10545	return VINF_SUCCESS;
10546	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10547	return rc;
10548
10549	#if 0
10550	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10551	* just sync the whole thing. */
10552	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10553	#else
10554	/* Aggressive state sync. for now. */
10555	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10556	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10557	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10558	#endif
10559	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10560	AssertRCReturn(rc, rc);
10561
10562	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10563	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10564	switch (uAccessType)
10565	{
10566	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10567	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10568	{
10569	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10570	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10571	{
10572	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10573	}
10574
10575	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10576	GCPhys &= PAGE_BASE_GC_MASK;
10577	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10578	PVM pVM = pVCpu->CTX_SUFF(pVM);
10579	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10580	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10581
10582	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10583	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10584	CPUMCTX2CORE(pMixedCtx), GCPhys);
10585	rc = VBOXSTRICTRC_VAL(rc2);
10586	Log4(("ApicAccess rc=%d\n", rc));
10587	if ( rc == VINF_SUCCESS
10588	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10589	\|\| rc == VERR_PAGE_NOT_PRESENT)
10590	{
10591	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10592	\| HM_CHANGED_GUEST_RSP
10593	\| HM_CHANGED_GUEST_RFLAGS
10594	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10595	rc = VINF_SUCCESS;
10596	}
10597	break;
10598	}
10599
10600	default:
10601	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10602	rc = VINF_EM_RAW_EMULATE_INSTR;
10603	break;
10604	}
10605
10606	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10607	return rc;
10608	}
10609
10610
10611	/**
10612	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10613	* VM-exit.
10614	*/
10615	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10616	{
10617	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10618
10619	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10620	if (pVmxTransient->fWasGuestDebugStateActive)
10621	{
10622	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10623	HMVMX_RETURN_UNEXPECTED_EXIT();
10624	}
10625
10626	int rc = VERR_INTERNAL_ERROR_5;
10627	if ( !DBGFIsStepping(pVCpu)
10628	&& !pVCpu->hm.s.fSingleInstruction
10629	&& !pVmxTransient->fWasHyperDebugStateActive)
10630	{
10631	/* Don't intercept MOV DRx and #DB any more. */
10632	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10633	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10634	AssertRCReturn(rc, rc);
10635
10636	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10637	{
10638	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10639	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10640	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10641	AssertRCReturn(rc, rc);
10642	#endif
10643	}
10644
10645	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10646	VMMRZCallRing3Disable(pVCpu);
10647	HM_DISABLE_PREEMPT_IF_NEEDED();
10648
10649	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10650	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10651	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10652
10653	HM_RESTORE_PREEMPT_IF_NEEDED();
10654	VMMRZCallRing3Enable(pVCpu);
10655
10656	#ifdef VBOX_WITH_STATISTICS
10657	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10658	AssertRCReturn(rc, rc);
10659	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10660	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10661	else
10662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10663	#endif
10664	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10665	return VINF_SUCCESS;
10666	}
10667
10668	/*
10669	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10670	* Update the segment registers and DR7 from the CPU.
10671	*/
10672	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10673	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10674	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10675	AssertRCReturn(rc, rc);
10676	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10677
10678	PVM pVM = pVCpu->CTX_SUFF(pVM);
10679	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10680	{
10681	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10682	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10683	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10684	if (RT_SUCCESS(rc))
10685	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10686	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10687	}
10688	else
10689	{
10690	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10691	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10692	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10693	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10694	}
10695
10696	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10697	if (RT_SUCCESS(rc))
10698	{
10699	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10700	AssertRCReturn(rc2, rc2);
10701	}
10702	return rc;
10703	}
10704
10705
10706	/**
10707	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10708	* Conditional VM-exit.
10709	*/
10710	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10711	{
10712	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10713	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10714
10715	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10716	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10717	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10718	return VINF_SUCCESS;
10719	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10720	return rc;
10721
10722	RTGCPHYS GCPhys = 0;
10723	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10724
10725	#if 0
10726	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10727	#else
10728	/* Aggressive state sync. for now. */
10729	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10730	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10731	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10732	#endif
10733	AssertRCReturn(rc, rc);
10734
10735	/*
10736	* If we succeed, resume guest execution.
10737	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10738	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10739	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10740	* weird case. See @bugref{6043}.
10741	*/
10742	PVM pVM = pVCpu->CTX_SUFF(pVM);
10743	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10744	rc = VBOXSTRICTRC_VAL(rc2);
10745	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10746	if ( rc == VINF_SUCCESS
10747	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10748	\|\| rc == VERR_PAGE_NOT_PRESENT)
10749	{
10750	/* Successfully handled MMIO operation. */
10751	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10752	\| HM_CHANGED_GUEST_RSP
10753	\| HM_CHANGED_GUEST_RFLAGS
10754	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10755	rc = VINF_SUCCESS;
10756	}
10757	return rc;
10758	}
10759
10760
10761	/**
10762	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10763	* VM-exit.
10764	*/
10765	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10766	{
10767	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10768	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10769
10770	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10771	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10772	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10773	return VINF_SUCCESS;
10774	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10775	return rc;
10776
10777	RTGCPHYS GCPhys = 0;
10778	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10779	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10780	#if 0
10781	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10782	#else
10783	/* Aggressive state sync. for now. */
10784	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10785	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10786	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10787	#endif
10788	AssertRCReturn(rc, rc);
10789
10790	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10791	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10792
10793	RTGCUINT uErrorCode = 0;
10794	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10795	uErrorCode \|= X86_TRAP_PF_ID;
10796	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10797	uErrorCode \|= X86_TRAP_PF_RW;
10798	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10799	uErrorCode \|= X86_TRAP_PF_P;
10800
10801	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10802
10803	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10804	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10805
10806	/* Handle the pagefault trap for the nested shadow table. */
10807	PVM pVM = pVCpu->CTX_SUFF(pVM);
10808	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10809	TRPMResetTrap(pVCpu);
10810
10811	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10812	if ( rc == VINF_SUCCESS
10813	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10814	\|\| rc == VERR_PAGE_NOT_PRESENT)
10815	{
10816	/* Successfully synced our nested page tables. */
10817	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10818	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10819	\| HM_CHANGED_GUEST_RSP
10820	\| HM_CHANGED_GUEST_RFLAGS);
10821	return VINF_SUCCESS;
10822	}
10823
10824	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10825	return rc;
10826	}
10827
10828	/** @} */
10829
10830	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10831	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10832	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10833
10834	/** @name VM-exit exception handlers.
10835	* @{
10836	*/
10837
10838	/**
10839	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10840	*/
10841	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10842	{
10843	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10844	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10845
10846	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10847	AssertRCReturn(rc, rc);
10848
10849	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10850	{
10851	/* Convert a #MF into a FERR -> IRQ 13. */
10852	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /uTagSrc/);
10853	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10854	AssertRCReturn(rc2, rc2);
10855	return rc;
10856	}
10857
10858	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10859	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10860	return rc;
10861	}
10862
10863
10864	/**
10865	* VM-exit exception handler for #BP (Breakpoint exception).
10866	*/
10867	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10868	{
10869	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10870	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10871
10872	/** @todo Try optimize this by not saving the entire guest state unless
10873	* really needed. */
10874	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10875	AssertRCReturn(rc, rc);
10876
10877	PVM pVM = pVCpu->CTX_SUFF(pVM);
10878	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10879	if (rc == VINF_EM_RAW_GUEST_TRAP)
10880	{
10881	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10882	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10883	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10884	AssertRCReturn(rc, rc);
10885
10886	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10887	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10888	}
10889
10890	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10891	return rc;
10892	}
10893
10894
10895	/**
10896	* VM-exit exception handler for #DB (Debug exception).
10897	*/
10898	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10899	{
10900	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10901	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10902	Log6(("XcptDB\n"));
10903
10904	/*
10905	* Get the DR6-like values from the exit qualification and pass it to DBGF
10906	* for processing.
10907	*/
10908	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10909	AssertRCReturn(rc, rc);
10910
10911	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10912	uint64_t uDR6 = X86_DR6_INIT_VAL;
10913	uDR6 \|= ( pVmxTransient->uExitQualification
10914	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10915
10916	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10917	if (rc == VINF_EM_RAW_GUEST_TRAP)
10918	{
10919	/*
10920	* The exception was for the guest. Update DR6, DR7.GD and
10921	* IA32_DEBUGCTL.LBR before forwarding it.
10922	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10923	*/
10924	VMMRZCallRing3Disable(pVCpu);
10925	HM_DISABLE_PREEMPT_IF_NEEDED();
10926
10927	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10928	pMixedCtx->dr[6] \|= uDR6;
10929	if (CPUMIsGuestDebugStateActive(pVCpu))
10930	ASMSetDR6(pMixedCtx->dr[6]);
10931
10932	HM_RESTORE_PREEMPT_IF_NEEDED();
10933	VMMRZCallRing3Enable(pVCpu);
10934
10935	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10936	AssertRCReturn(rc, rc);
10937
10938	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10939	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10940
10941	/* Paranoia. */
10942	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10943	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10944
10945	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10946	AssertRCReturn(rc, rc);
10947
10948	/*
10949	* Raise #DB in the guest.
10950	*/
10951	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10952	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10953	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10954	AssertRCReturn(rc, rc);
10955	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10956	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10957	return VINF_SUCCESS;
10958	}
10959
10960	/*
10961	* Not a guest trap, must be a hypervisor related debug event then.
10962	* Update DR6 in case someone is interested in it.
10963	*/
10964	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10965	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10966	CPUMSetHyperDR6(pVCpu, uDR6);
10967
10968	return rc;
10969	}
10970
10971
10972	/**
10973	* VM-exit exception handler for #NM (Device-not-available exception: floating
10974	* point exception).
10975	*/
10976	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10977	{
10978	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10979
10980	/* We require CR0 and EFER. EFER is always up-to-date. */
10981	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10982	AssertRCReturn(rc, rc);
10983
10984	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10985	VMMRZCallRing3Disable(pVCpu);
10986	HM_DISABLE_PREEMPT_IF_NEEDED();
10987
10988	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10989	if (pVmxTransient->fWasGuestFPUStateActive)
10990	{
10991	rc = VINF_EM_RAW_GUEST_TRAP;
10992	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10993	}
10994	else
10995	{
10996	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10997	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10998	#endif
10999	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11000	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11001	}
11002
11003	HM_RESTORE_PREEMPT_IF_NEEDED();
11004	VMMRZCallRing3Enable(pVCpu);
11005
11006	if (rc == VINF_SUCCESS)
11007	{
11008	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11009	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11010	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11011	pVCpu->hm.s.fUseGuestFpu = true;
11012	}
11013	else
11014	{
11015	/* Forward #NM to the guest. */
11016	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11017	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11018	AssertRCReturn(rc, rc);
11019	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11020	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11021	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11022	}
11023
11024	return VINF_SUCCESS;
11025	}
11026
11027
11028	/**
11029	* VM-exit exception handler for #GP (General-protection exception).
11030	*
11031	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11032	*/
11033	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11034	{
11035	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11036	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11037
11038	int rc = VERR_INTERNAL_ERROR_5;
11039	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11040	{
11041	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11042	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11043	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11044	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11045	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11046	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11047	AssertRCReturn(rc, rc);
11048	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11049	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11050	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11051	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11052	return rc;
11053	#else
11054	/* We don't intercept #GP. */
11055	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11056	NOREF(pVmxTransient);
11057	return VERR_VMX_UNEXPECTED_EXCEPTION;
11058	#endif
11059	}
11060
11061	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11062	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11063
11064	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11065	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11066	AssertRCReturn(rc, rc);
11067
11068	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11069	uint32_t cbOp = 0;
11070	PVM pVM = pVCpu->CTX_SUFF(pVM);
11071	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11072	if (RT_SUCCESS(rc))
11073	{
11074	rc = VINF_SUCCESS;
11075	Assert(cbOp == pDis->cbInstr);
11076	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11077	switch (pDis->pCurInstr->uOpcode)
11078	{
11079	case OP_CLI:
11080	{
11081	pMixedCtx->eflags.Bits.u1IF = 0;
11082	pMixedCtx->rip += pDis->cbInstr;
11083	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11084	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11085	break;
11086	}
11087
11088	case OP_STI:
11089	{
11090	pMixedCtx->eflags.Bits.u1IF = 1;
11091	pMixedCtx->rip += pDis->cbInstr;
11092	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11093	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11094	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11095	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11096	break;
11097	}
11098
11099	case OP_HLT:
11100	{
11101	rc = VINF_EM_HALT;
11102	pMixedCtx->rip += pDis->cbInstr;
11103	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11104	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11105	break;
11106	}
11107
11108	case OP_POPF:
11109	{
11110	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11111	uint32_t cbParm = 0;
11112	uint32_t uMask = 0;
11113	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11114	{
11115	cbParm = 4;
11116	uMask = 0xffffffff;
11117	}
11118	else
11119	{
11120	cbParm = 2;
11121	uMask = 0xffff;
11122	}
11123
11124	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11125	RTGCPTR GCPtrStack = 0;
11126	X86EFLAGS Eflags;
11127	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11128	&GCPtrStack);
11129	if (RT_SUCCESS(rc))
11130	{
11131	Assert(sizeof(Eflags.u32) >= cbParm);
11132	Eflags.u32 = 0;
11133	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11134	}
11135	if (RT_FAILURE(rc))
11136	{
11137	rc = VERR_EM_INTERPRETER;
11138	break;
11139	}
11140	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11141	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11142	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11143	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11144	pMixedCtx->esp += cbParm;
11145	pMixedCtx->esp &= uMask;
11146	pMixedCtx->rip += pDis->cbInstr;
11147
11148	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11149	\| HM_CHANGED_GUEST_RSP
11150	\| HM_CHANGED_GUEST_RFLAGS);
11151	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11152	break;
11153	}
11154
11155	case OP_PUSHF:
11156	{
11157	uint32_t cbParm = 0;
11158	uint32_t uMask = 0;
11159	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11160	{
11161	cbParm = 4;
11162	uMask = 0xffffffff;
11163	}
11164	else
11165	{
11166	cbParm = 2;
11167	uMask = 0xffff;
11168	}
11169
11170	/* Get the stack pointer & push the contents of eflags onto the stack. */
11171	RTGCPTR GCPtrStack = 0;
11172	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11173	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11174	if (RT_FAILURE(rc))
11175	{
11176	rc = VERR_EM_INTERPRETER;
11177	break;
11178	}
11179	X86EFLAGS Eflags = pMixedCtx->eflags;
11180	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11181	Eflags.Bits.u1RF = 0;
11182	Eflags.Bits.u1VM = 0;
11183
11184	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11185	if (RT_FAILURE(rc))
11186	{
11187	rc = VERR_EM_INTERPRETER;
11188	break;
11189	}
11190	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11191	pMixedCtx->esp -= cbParm;
11192	pMixedCtx->esp &= uMask;
11193	pMixedCtx->rip += pDis->cbInstr;
11194	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11195	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11196	break;
11197	}
11198
11199	case OP_IRET:
11200	{
11201	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11202	* instruction reference. */
11203	RTGCPTR GCPtrStack = 0;
11204	uint32_t uMask = 0xffff;
11205	uint16_t aIretFrame[3];
11206	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11207	{
11208	rc = VERR_EM_INTERPRETER;
11209	break;
11210	}
11211	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11212	&GCPtrStack);
11213	if (RT_SUCCESS(rc))
11214	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11215	if (RT_FAILURE(rc))
11216	{
11217	rc = VERR_EM_INTERPRETER;
11218	break;
11219	}
11220	pMixedCtx->eip = 0;
11221	pMixedCtx->ip = aIretFrame[0];
11222	pMixedCtx->cs.Sel = aIretFrame[1];
11223	pMixedCtx->cs.ValidSel = aIretFrame[1];
11224	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11225	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11226	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11227	pMixedCtx->sp += sizeof(aIretFrame);
11228	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11229	\| HM_CHANGED_GUEST_SEGMENT_REGS
11230	\| HM_CHANGED_GUEST_RSP
11231	\| HM_CHANGED_GUEST_RFLAGS);
11232	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11233	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11234	break;
11235	}
11236
11237	case OP_INT:
11238	{
11239	uint16_t uVector = pDis->Param1.uValue & 0xff;
11240	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11241	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11242	break;
11243	}
11244
11245	case OP_INTO:
11246	{
11247	if (pMixedCtx->eflags.Bits.u1OF)
11248	{
11249	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11250	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11251	}
11252	break;
11253	}
11254
11255	default:
11256	{
11257	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11258	EMCODETYPE_SUPERVISOR);
11259	rc = VBOXSTRICTRC_VAL(rc2);
11260	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11261	Log4(("#GP rc=%Rrc\n", rc));
11262	break;
11263	}
11264	}
11265	}
11266	else
11267	rc = VERR_EM_INTERPRETER;
11268
11269	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11270	("#GP Unexpected rc=%Rrc\n", rc));
11271	return rc;
11272	}
11273
11274
11275	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11276	/**
11277	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11278	* the exception reported in the VMX transient structure back into the VM.
11279	*
11280	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11281	* up-to-date.
11282	*/
11283	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11284	{
11285	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11286
11287	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11288	hmR0VmxCheckExitDueToEventDelivery(). */
11289	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11290	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11291	AssertRCReturn(rc, rc);
11292	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11293
11294	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11295	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11296	return VINF_SUCCESS;
11297	}
11298	#endif
11299
11300
11301	/**
11302	* VM-exit exception handler for #PF (Page-fault exception).
11303	*/
11304	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11305	{
11306	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11307	PVM pVM = pVCpu->CTX_SUFF(pVM);
11308	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11309	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11310	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11311	AssertRCReturn(rc, rc);
11312
11313	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11314	if (pVM->hm.s.fNestedPaging)
11315	{
11316	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11317	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11318	{
11319	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11320	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11321	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11322	}
11323	else
11324	{
11325	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11326	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11327	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11328	}
11329	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11330	return rc;
11331	}
11332	#else
11333	Assert(!pVM->hm.s.fNestedPaging);
11334	NOREF(pVM);
11335	#endif
11336
11337	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11338	AssertRCReturn(rc, rc);
11339
11340	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11341	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11342
11343	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11344	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11345	(RTGCPTR)pVmxTransient->uExitQualification);
11346
11347	Log4(("#PF: rc=%Rrc\n", rc));
11348	if (rc == VINF_SUCCESS)
11349	{
11350	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11351	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11352	* memory? We don't update the whole state here... */
11353	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11354	\| HM_CHANGED_GUEST_RSP
11355	\| HM_CHANGED_GUEST_RFLAGS
11356	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11357	TRPMResetTrap(pVCpu);
11358	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11359	return rc;
11360	}
11361	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11362	{
11363	if (!pVmxTransient->fVectoringPF)
11364	{
11365	/* It's a guest page fault and needs to be reflected to the guest. */
11366	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11367	TRPMResetTrap(pVCpu);
11368	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11369	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11370	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11371	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11372	}
11373	else
11374	{
11375	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11376	TRPMResetTrap(pVCpu);
11377	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11378	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11379	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11380	}
11381
11382	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11383	return VINF_SUCCESS;
11384	}
11385
11386	TRPMResetTrap(pVCpu);
11387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11388	return rc;
11389	}
11390
11391	/** @} */
11392

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49721

Download in other formats: