HMVMXR0.cpp@ 50275

Last change on this file since 50275 was 50275, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Fix single-stepping in real-on-v86 mode for certain emulated instructions. Added todo for the rest.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 474.0 KB

Line
1	/* $Id: HMVMXR0.cpp 50275 2014-01-29 17:27:37Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu Pointer to the VMCPU.
1240	* @param uMsr The MSR.
1241	* @param uGuestMsr Value of the guest MSR.
1242	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1243	* necessary.
1244	*/
1245	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1246	{
1247	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1248	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1249	uint32_t i;
1250	for (i = 0; i < cMsrs; i++)
1251	{
1252	if (pGuestMsr->u32Msr == uMsr)
1253	break;
1254	pGuestMsr++;
1255	}
1256
1257	bool fAdded = false;
1258	if (i == cMsrs)
1259	{
1260	++cMsrs;
1261	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1262	AssertRCReturn(rc, rc);
1263
1264	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1265	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1266	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1267
1268	fAdded = true;
1269	}
1270
1271	/* Update the MSR values in the auto-load/store MSR area. */
1272	pGuestMsr->u32Msr = uMsr;
1273	pGuestMsr->u64Value = uGuestMsrValue;
1274
1275	/* Create/update the MSR slot in the host MSR area. */
1276	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1277	pHostMsr += i;
1278	pHostMsr->u32Msr = uMsr;
1279
1280	/*
1281	* Update the host MSR only when requested by the caller AND when we're
1282	* adding it to the auto-load/store area. Otherwise, it would have been
1283	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1284	*/
1285	if ( fAdded
1286	&& fUpdateHostMsr)
1287	{
1288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1290	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1291	}
1292
1293	return VINF_SUCCESS;
1294	}
1295
1296
1297	/**
1298	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1299	* auto-load/store MSR area in the VMCS.
1300	*
1301	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1302	* area.
1303	*
1304	* @returns VBox status code.
1305	* @param pVCpu Pointer to the VMCPU.
1306	* @param uMsr The MSR.
1307	*/
1308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1309	{
1310	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1311	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1312	for (uint32_t i = 0; i < cMsrs; i++)
1313	{
1314	/* Find the MSR. */
1315	if (pGuestMsr->u32Msr == uMsr)
1316	{
1317	/* If it's the last MSR, simply reduce the count. */
1318	if (i == cMsrs - 1)
1319	{
1320	--cMsrs;
1321	break;
1322	}
1323
1324	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1325	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326	pLastGuestMsr += cMsrs;
1327	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1328	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1329
1330	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332	pLastHostMsr += cMsrs;
1333	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1334	pHostMsr->u64Value = pLastHostMsr->u64Value;
1335	--cMsrs;
1336	break;
1337	}
1338	pGuestMsr++;
1339	}
1340
1341	/* Update the VMCS if the count changed (meaning the MSR was found). */
1342	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1343	{
1344	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1345	AssertRCReturn(rc, rc);
1346
1347	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1348	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1349	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1350	}
1351
1352	return VINF_SUCCESS;
1353	}
1354
1355
1356	/**
1357	* Checks if the specified guest MSR is part of the auto-load/store area in
1358	* the VMCS.
1359	*
1360	* @returns true if found, false otherwise.
1361	* @param pVCpu Pointer to the VMCPU.
1362	* @param uMsr The MSR to find.
1363	*/
1364	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1365	{
1366	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368
1369	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1370	{
1371	if (pGuestMsr->u32Msr == uMsr)
1372	return true;
1373	}
1374	return false;
1375	}
1376
1377
1378	/**
1379	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1380	*
1381	* @param pVCpu Pointer to the VMCPU.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1386	{
1387	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1388	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1389	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1390	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1391
1392	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1393	{
1394	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1395	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1396	}
1397
1398	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1399	}
1400
1401
1402	#if HC_ARCH_BITS == 64
1403	/**
1404	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1405	* perform lazy restoration of the host MSRs while leaving VT-x.
1406	*
1407	* @param pVCpu Pointer to the VMCPU.
1408	*
1409	* @remarks No-long-jump zone!!!
1410	*/
1411	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1412	{
1413	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1414
1415	/*
1416	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1417	*/
1418	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1419	{
1420	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1421	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1422	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1423	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1424	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1425	}
1426	}
1427
1428
1429	/**
1430	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1431	* lazily while leaving VT-x.
1432	*
1433	* @returns true if it does, false otherwise.
1434	* @param pVCpu Pointer to the VMCPU.
1435	* @param uMsr The MSR to check.
1436	*/
1437	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1438	{
1439	NOREF(pVCpu);
1440	switch (uMsr)
1441	{
1442	case MSR_K8_LSTAR:
1443	case MSR_K6_STAR:
1444	case MSR_K8_SF_MASK:
1445	case MSR_K8_KERNEL_GS_BASE:
1446	return true;
1447	}
1448	return false;
1449	}
1450
1451
1452	/**
1453	* Saves a set of guests MSRs back into the guest-CPU context.
1454	*
1455	* @param pVCpu Pointer to the VMCPU.
1456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1457	* out-of-sync. Make sure to update the required fields
1458	* before using them.
1459	*
1460	* @remarks No-long-jump zone!!!
1461	*/
1462	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1463	{
1464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1465	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1466
1467	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1468	{
1469	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1470	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1471	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1472	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1473	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1474	}
1475	}
1476
1477
1478	/**
1479	* Loads a set of guests MSRs to allow read/passthru to the guest.
1480	*
1481	* The name of this function is slightly confusing. This function does NOT
1482	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1483	* common prefix for functions dealing with "lazy restoration" of the shared
1484	* MSRs.
1485	*
1486	* @param pVCpu Pointer to the VMCPU.
1487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1488	* out-of-sync. Make sure to update the required fields
1489	* before using them.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1497
1498	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1499	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1500	{
1501	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1502	do { \
1503	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1504	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1505	else \
1506	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1507	} while (0)
1508
1509	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1510	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1511	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1513	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1514	}
1515	else
1516	{
1517	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1518	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1519	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1520	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1521	}
1522	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1523	}
1524
1525
1526	/**
1527	* Performs lazy restoration of the set of host MSRs if they were previously
1528	* loaded with guest MSR values.
1529	*
1530	* @param pVCpu Pointer to the VMCPU.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	* @remarks The guest MSRs should have been saved back into the guest-CPU
1534	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1535	*/
1536	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1537	{
1538	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1539	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1540
1541	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1542	{
1543	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1544	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1545	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1546	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1547	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1548	}
1549	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1550	}
1551	#endif /* HC_ARCH_BITS == 64 */
1552
1553
1554	#ifdef VBOX_STRICT
1555	/**
1556	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1557	* VMCS are correct.
1558	*
1559	* @param pVCpu Pointer to the VMCPU.
1560	*/
1561	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1562	{
1563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564
1565	/* Verify MSR counts in the VMCS are what we think it should be. */
1566	uint32_t cMsrs;
1567	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1568	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1569
1570	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1571	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1574	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1577	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1578	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1579	{
1580	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1581	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1582	pGuestMsr->u32Msr));
1583
1584	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1585	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1586	pHostMsr->u64Value, u64Msr));
1587
1588	/* Verify that the permissions are as expected in the MSR bitmap. */
1589	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1590	{
1591	VMXMSREXITREAD enmRead;
1592	VMXMSREXITWRITE enmWrite;
1593	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1594	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1595	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1596	pGuestMsr->u32Msr));
1597	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1598	pGuestMsr->u32Msr));
1599	}
1600	}
1601	}
1602	# endif /* VBOX_STRICT */
1603
1604
1605	/**
1606	* Flushes the TLB using EPT.
1607	*
1608	* @returns VBox status code.
1609	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1610	* enmFlush).
1611	* @param enmFlush Type of flush.
1612	*
1613	* @remarks Caller is responsible for making sure this function is called only
1614	* when NestedPaging is supported and providing @a enmFlush that is
1615	* supported by the CPU.
1616	* @remarks Can be called with interrupts disabled.
1617	*/
1618	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1619	{
1620	uint64_t au64Descriptor[2];
1621	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1622	au64Descriptor[0] = 0;
1623	else
1624	{
1625	Assert(pVCpu);
1626	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1627	}
1628	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1629
1630	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1631	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1632	rc));
1633	if ( RT_SUCCESS(rc)
1634	&& pVCpu)
1635	{
1636	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1637	}
1638	}
1639
1640
1641	/**
1642	* Flushes the TLB using VPID.
1643	*
1644	* @returns VBox status code.
1645	* @param pVM Pointer to the VM.
1646	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1647	* enmFlush).
1648	* @param enmFlush Type of flush.
1649	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1650	* on @a enmFlush).
1651	*
1652	* @remarks Can be called with interrupts disabled.
1653	*/
1654	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1655	{
1656	NOREF(pVM);
1657	AssertPtr(pVM);
1658	Assert(pVM->hm.s.vmx.fVpid);
1659
1660	uint64_t au64Descriptor[2];
1661	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1662	{
1663	au64Descriptor[0] = 0;
1664	au64Descriptor[1] = 0;
1665	}
1666	else
1667	{
1668	AssertPtr(pVCpu);
1669	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1670	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1671	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1672	au64Descriptor[1] = GCPtr;
1673	}
1674
1675	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1676	AssertMsg(rc == VINF_SUCCESS,
1677	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1678	if ( RT_SUCCESS(rc)
1679	&& pVCpu)
1680	{
1681	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1682	}
1683	}
1684
1685
1686	/**
1687	* Invalidates a guest page by guest virtual address. Only relevant for
1688	* EPT/VPID, otherwise there is nothing really to invalidate.
1689	*
1690	* @returns VBox status code.
1691	* @param pVM Pointer to the VM.
1692	* @param pVCpu Pointer to the VMCPU.
1693	* @param GCVirt Guest virtual address of the page to invalidate.
1694	*/
1695	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1696	{
1697	AssertPtr(pVM);
1698	AssertPtr(pVCpu);
1699	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1700
1701	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1702	if (!fFlushPending)
1703	{
1704	/*
1705	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1706	* See @bugref{6043} and @bugref{6177}.
1707	*
1708	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1709	* function maybe called in a loop with individual addresses.
1710	*/
1711	if (pVM->hm.s.vmx.fVpid)
1712	{
1713	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1714	{
1715	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1716	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1717	}
1718	else
1719	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1720	}
1721	else if (pVM->hm.s.fNestedPaging)
1722	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723	}
1724
1725	return VINF_SUCCESS;
1726	}
1727
1728
1729	/**
1730	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1731	* otherwise there is nothing really to invalidate.
1732	*
1733	* @returns VBox status code.
1734	* @param pVM Pointer to the VM.
1735	* @param pVCpu Pointer to the VMCPU.
1736	* @param GCPhys Guest physical address of the page to invalidate.
1737	*/
1738	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1739	{
1740	NOREF(pVM); NOREF(GCPhys);
1741	LogFlowFunc(("%RGp\n", GCPhys));
1742
1743	/*
1744	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1745	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1746	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1747	*/
1748	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1749	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1750	return VINF_SUCCESS;
1751	}
1752
1753
1754	/**
1755	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1756	* case where neither EPT nor VPID is supported by the CPU.
1757	*
1758	* @param pVM Pointer to the VM.
1759	* @param pVCpu Pointer to the VMCPU.
1760	* @param pCpu Pointer to the global HM struct.
1761	*
1762	* @remarks Called with interrupts disabled.
1763	*/
1764	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1765	{
1766	AssertPtr(pVCpu);
1767	AssertPtr(pCpu);
1768	NOREF(pVM);
1769
1770	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1771
1772	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1773	#if 0
1774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1775	pVCpu->hm.s.TlbShootdown.cPages = 0;
1776	#endif
1777
1778	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1779	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1780	pVCpu->hm.s.fForceTLBFlush = false;
1781	return;
1782	}
1783
1784
1785	/**
1786	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1787	*
1788	* @param pVM Pointer to the VM.
1789	* @param pVCpu Pointer to the VMCPU.
1790	* @param pCpu Pointer to the global HM CPU struct.
1791	* @remarks All references to "ASID" in this function pertains to "VPID" in
1792	* Intel's nomenclature. The reason is, to avoid confusion in compare
1793	* statements since the host-CPU copies are named "ASID".
1794	*
1795	* @remarks Called with interrupts disabled.
1796	*/
1797	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1798	{
1799	#ifdef VBOX_WITH_STATISTICS
1800	bool fTlbFlushed = false;
1801	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1802	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1803	if (!fTlbFlushed) \
1804	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1805	} while (0)
1806	#else
1807	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1808	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1809	#endif
1810
1811	AssertPtr(pVM);
1812	AssertPtr(pCpu);
1813	AssertPtr(pVCpu);
1814	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1815	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1816	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1817
1818
1819	/*
1820	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1821	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1822	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1823	*/
1824	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1825	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1826	{
1827	++pCpu->uCurrentAsid;
1828	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1829	{
1830	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1831	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1832	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1833	}
1834
1835	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1836	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1837	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1838
1839	/*
1840	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1841	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1842	*/
1843	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1844	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1845	HMVMX_SET_TAGGED_TLB_FLUSHED();
1846	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1847	}
1848
1849	/* Check for explicit TLB shootdowns. */
1850	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1851	{
1852	/*
1853	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1854	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1855	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1856	* but not guest-physical mappings.
1857	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1858	*/
1859	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1860	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1861	HMVMX_SET_TAGGED_TLB_FLUSHED();
1862	}
1863
1864	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1865	* where it is commented out. Support individual entry flushing
1866	* someday. */
1867	#if 0
1868	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1869	{
1870	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1871
1872	/*
1873	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1874	* as supported by the CPU.
1875	*/
1876	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1877	{
1878	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1879	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1880	}
1881	else
1882	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1883
1884	HMVMX_SET_TAGGED_TLB_FLUSHED();
1885	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1886	pVCpu->hm.s.TlbShootdown.cPages = 0;
1887	}
1888	#endif
1889
1890	pVCpu->hm.s.fForceTLBFlush = false;
1891
1892	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1893
1894	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1895	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1896	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1897	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1898	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1899	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1900	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1901	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1902
1903	/* Update VMCS with the VPID. */
1904	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1905	AssertRC(rc);
1906
1907	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1908	}
1909
1910
1911	/**
1912	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	* @param pVCpu Pointer to the VMCPU.
1917	* @param pCpu Pointer to the global HM CPU struct.
1918	*
1919	* @remarks Called with interrupts disabled.
1920	*/
1921	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922	{
1923	AssertPtr(pVM);
1924	AssertPtr(pVCpu);
1925	AssertPtr(pCpu);
1926	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1927	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1928
1929	/*
1930	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1931	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1932	*/
1933	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1934	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1935	{
1936	pVCpu->hm.s.fForceTLBFlush = true;
1937	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1938	}
1939
1940	/* Check for explicit TLB shootdown flushes. */
1941	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1942	{
1943	pVCpu->hm.s.fForceTLBFlush = true;
1944	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1945	}
1946
1947	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1948	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1949
1950	if (pVCpu->hm.s.fForceTLBFlush)
1951	{
1952	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1953	pVCpu->hm.s.fForceTLBFlush = false;
1954	}
1955	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1956	* where it is commented out. Support individual entry flushing
1957	* someday. */
1958	#if 0
1959	else
1960	{
1961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1962	{
1963	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1964	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1966	}
1967	else
1968	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1969
1970	pVCpu->hm.s.TlbShootdown.cPages = 0;
1971	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1972	}
1973	#endif
1974	}
1975
1976
1977	/**
1978	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1979	*
1980	* @returns VBox status code.
1981	* @param pVM Pointer to the VM.
1982	* @param pVCpu Pointer to the VMCPU.
1983	* @param pCpu Pointer to the global HM CPU struct.
1984	*
1985	* @remarks Called with interrupts disabled.
1986	*/
1987	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1988	{
1989	AssertPtr(pVM);
1990	AssertPtr(pVCpu);
1991	AssertPtr(pCpu);
1992	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1993	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1994
1995	/*
1996	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1997	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1998	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1999	*/
2000	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2001	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2002	{
2003	pVCpu->hm.s.fForceTLBFlush = true;
2004	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2005	}
2006
2007	/* Check for explicit TLB shootdown flushes. */
2008	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2009	{
2010	/*
2011	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2012	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2013	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2014	*/
2015	pVCpu->hm.s.fForceTLBFlush = true;
2016	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2017	}
2018
2019	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2020	if (pVCpu->hm.s.fForceTLBFlush)
2021	{
2022	++pCpu->uCurrentAsid;
2023	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2024	{
2025	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2026	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2027	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2028	}
2029
2030	pVCpu->hm.s.fForceTLBFlush = false;
2031	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2032	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2033	if (pCpu->fFlushAsidBeforeUse)
2034	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2035	}
2036	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2037	* where it is commented out. Support individual entry flushing
2038	* someday. */
2039	#if 0
2040	else
2041	{
2042	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2043	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2044	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2045	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2046
2047	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2048	{
2049	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2050	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2051	{
2052	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2053	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2054	}
2055	else
2056	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2057
2058	pVCpu->hm.s.TlbShootdown.cPages = 0;
2059	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2060	}
2061	else
2062	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2063	}
2064	#endif
2065
2066	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2067	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2068	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2069	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2070	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2071	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2072
2073	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2074	AssertRC(rc);
2075	}
2076
2077
2078	/**
2079	* Flushes the guest TLB entry based on CPU capabilities.
2080	*
2081	* @param pVCpu Pointer to the VMCPU.
2082	* @param pCpu Pointer to the global HM CPU struct.
2083	*/
2084	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2087	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2088	#endif
2089	PVM pVM = pVCpu->CTX_SUFF(pVM);
2090	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2091	{
2092	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2093	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2094	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2095	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2096	default:
2097	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2098	break;
2099	}
2100
2101	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2102	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2103
2104	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2105	}
2106
2107
2108	/**
2109	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2110	* TLB entries from the host TLB before VM-entry.
2111	*
2112	* @returns VBox status code.
2113	* @param pVM Pointer to the VM.
2114	*/
2115	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2116	{
2117	/*
2118	* Determine optimal flush type for Nested Paging.
2119	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2120	* guest execution (see hmR3InitFinalizeR0()).
2121	*/
2122	if (pVM->hm.s.fNestedPaging)
2123	{
2124	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2125	{
2126	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2127	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2128	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2129	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2130	else
2131	{
2132	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2133	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2134	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2135	}
2136
2137	/* Make sure the write-back cacheable memory type for EPT is supported. */
2138	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2139	{
2140	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2143	}
2144	}
2145	else
2146	{
2147	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2148	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2149	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2150	}
2151	}
2152
2153	/*
2154	* Determine optimal flush type for VPID.
2155	*/
2156	if (pVM->hm.s.vmx.fVpid)
2157	{
2158	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2159	{
2160	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2161	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2162	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2164	else
2165	{
2166	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2167	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2169	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2170	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2171	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2172	pVM->hm.s.vmx.fVpid = false;
2173	}
2174	}
2175	else
2176	{
2177	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2178	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2179	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2180	pVM->hm.s.vmx.fVpid = false;
2181	}
2182	}
2183
2184	/*
2185	* Setup the handler for flushing tagged-TLBs.
2186	*/
2187	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2188	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2189	else if (pVM->hm.s.fNestedPaging)
2190	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2191	else if (pVM->hm.s.vmx.fVpid)
2192	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2193	else
2194	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2195	return VINF_SUCCESS;
2196	}
2197
2198
2199	/**
2200	* Sets up pin-based VM-execution controls in the VMCS.
2201	*
2202	* @returns VBox status code.
2203	* @param pVM Pointer to the VM.
2204	* @param pVCpu Pointer to the VMCPU.
2205	*/
2206	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2207	{
2208	AssertPtr(pVM);
2209	AssertPtr(pVCpu);
2210
2211	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2212	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2213
2214	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2215	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2216	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2217
2218	/* Enable the VMX preemption timer. */
2219	if (pVM->hm.s.vmx.fUsePreemptTimer)
2220	{
2221	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2222	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2223	}
2224
2225	if ((val & zap) != val)
2226	{
2227	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2228	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2229	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2230	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2231	}
2232
2233	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2234	AssertRCReturn(rc, rc);
2235
2236	/* Update VCPU with the currently set pin-based VM-execution controls. */
2237	pVCpu->hm.s.vmx.u32PinCtls = val;
2238	return rc;
2239	}
2240
2241
2242	/**
2243	* Sets up processor-based VM-execution controls in the VMCS.
2244	*
2245	* @returns VBox status code.
2246	* @param pVM Pointer to the VM.
2247	* @param pVMCPU Pointer to the VMCPU.
2248	*/
2249	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2250	{
2251	AssertPtr(pVM);
2252	AssertPtr(pVCpu);
2253
2254	int rc = VERR_INTERNAL_ERROR_5;
2255	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2256	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2257
2258	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2259	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2260	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2261	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2265
2266	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2267	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2268	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2269	{
2270	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2271	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2272	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2273	}
2274
2275	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2276	if (!pVM->hm.s.fNestedPaging)
2277	{
2278	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2279	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2280	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2281	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2282	}
2283
2284	/* Use TPR shadowing if supported by the CPU. */
2285	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2286	{
2287	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2288	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2289	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2290	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2291	AssertRCReturn(rc, rc);
2292
2293	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2294	/* CR8 writes causes a VM-exit based on TPR threshold. */
2295	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2296	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2297	}
2298	else
2299	{
2300	/*
2301	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2302	* Set this control only for 64-bit guests.
2303	*/
2304	if (pVM->hm.s.fAllow64BitGuests)
2305	{
2306	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2307	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2308	}
2309	}
2310
2311	/* Use MSR-bitmaps if supported by the CPU. */
2312	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2313	{
2314	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2315
2316	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2317	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2318	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2319	AssertRCReturn(rc, rc);
2320
2321	/*
2322	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2323	* automatically as dedicated fields in the VMCS.
2324	*/
2325	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2326	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2327	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330
2331	#if HC_ARCH_BITS == 64
2332	/*
2333	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2334	*/
2335	if (pVM->hm.s.fAllow64BitGuests)
2336	{
2337	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2339	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	}
2342	#endif
2343	}
2344
2345	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2346	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2347	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2348
2349	if ((val & zap) != val)
2350	{
2351	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2352	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2353	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2354	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2355	}
2356
2357	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2358	AssertRCReturn(rc, rc);
2359
2360	/* Update VCPU with the currently set processor-based VM-execution controls. */
2361	pVCpu->hm.s.vmx.u32ProcCtls = val;
2362
2363	/*
2364	* Secondary processor-based VM-execution controls.
2365	*/
2366	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2367	{
2368	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2373
2374	if (pVM->hm.s.fNestedPaging)
2375	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2376	else
2377	{
2378	/*
2379	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2380	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2381	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2382	*/
2383	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2385	}
2386
2387	if (pVM->hm.s.vmx.fVpid)
2388	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2389
2390	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2392
2393	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2394	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2395	* done dynamically. */
2396	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2397	{
2398	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2399	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2400	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2401	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2402	AssertRCReturn(rc, rc);
2403	}
2404
2405	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2406	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2407
2408	if ((val & zap) != val)
2409	{
2410	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2411	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2412	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2413	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2414	}
2415
2416	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2417	AssertRCReturn(rc, rc);
2418
2419	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2420	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2421	}
2422	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2423	{
2424	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2425	"available\n"));
2426	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2427	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2428	}
2429
2430	return VINF_SUCCESS;
2431	}
2432
2433
2434	/**
2435	* Sets up miscellaneous (everything other than Pin & Processor-based
2436	* VM-execution) control fields in the VMCS.
2437	*
2438	* @returns VBox status code.
2439	* @param pVM Pointer to the VM.
2440	* @param pVCpu Pointer to the VMCPU.
2441	*/
2442	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2443	{
2444	NOREF(pVM);
2445	AssertPtr(pVM);
2446	AssertPtr(pVCpu);
2447
2448	int rc = VERR_GENERAL_FAILURE;
2449
2450	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2451	#if 0
2452	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2453	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2454	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2455
2456	/*
2457	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2458	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2459	* We thus use the exception bitmap to control it rather than use both.
2460	*/
2461	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2463
2464	/** @todo Explore possibility of using IO-bitmaps. */
2465	/* All IO & IOIO instructions cause VM-exits. */
2466	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2467	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2468
2469	/* Initialize the MSR-bitmap area. */
2470	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2472	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2473	#endif
2474
2475	/* Setup MSR auto-load/store area. */
2476	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2477	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2478	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2479	AssertRCReturn(rc, rc);
2480	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2481	AssertRCReturn(rc, rc);
2482
2483	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2484	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2485	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2486	AssertRCReturn(rc, rc);
2487
2488	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2489	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2490	AssertRCReturn(rc, rc);
2491
2492	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2493	#if 0
2494	/* Setup debug controls */
2495	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2496	AssertRCReturn(rc, rc);
2497	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2498	AssertRCReturn(rc, rc);
2499	#endif
2500
2501	return rc;
2502	}
2503
2504
2505	/**
2506	* Sets up the initial exception bitmap in the VMCS based on static conditions
2507	* (i.e. conditions that cannot ever change after starting the VM).
2508	*
2509	* @returns VBox status code.
2510	* @param pVM Pointer to the VM.
2511	* @param pVCpu Pointer to the VMCPU.
2512	*/
2513	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2514	{
2515	AssertPtr(pVM);
2516	AssertPtr(pVCpu);
2517
2518	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2519
2520	uint32_t u32XcptBitmap = 0;
2521
2522	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2523	if (!pVM->hm.s.fNestedPaging)
2524	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2525
2526	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2527	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2528	AssertRCReturn(rc, rc);
2529	return rc;
2530	}
2531
2532
2533	/**
2534	* Sets up the initial guest-state mask. The guest-state mask is consulted
2535	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2536	* for the nested virtualization case (as it would cause a VM-exit).
2537	*
2538	* @param pVCpu Pointer to the VMCPU.
2539	*/
2540	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2541	{
2542	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2543	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2544	return VINF_SUCCESS;
2545	}
2546
2547
2548	/**
2549	* Does per-VM VT-x initialization.
2550	*
2551	* @returns VBox status code.
2552	* @param pVM Pointer to the VM.
2553	*/
2554	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2555	{
2556	LogFlowFunc(("pVM=%p\n", pVM));
2557
2558	int rc = hmR0VmxStructsAlloc(pVM);
2559	if (RT_FAILURE(rc))
2560	{
2561	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2562	return rc;
2563	}
2564
2565	return VINF_SUCCESS;
2566	}
2567
2568
2569	/**
2570	* Does per-VM VT-x termination.
2571	*
2572	* @returns VBox status code.
2573	* @param pVM Pointer to the VM.
2574	*/
2575	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2576	{
2577	LogFlowFunc(("pVM=%p\n", pVM));
2578
2579	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2580	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2581	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2582	#endif
2583	hmR0VmxStructsFree(pVM);
2584	return VINF_SUCCESS;
2585	}
2586
2587
2588	/**
2589	* Sets up the VM for execution under VT-x.
2590	* This function is only called once per-VM during initialization.
2591	*
2592	* @returns VBox status code.
2593	* @param pVM Pointer to the VM.
2594	*/
2595	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2596	{
2597	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2598	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2599
2600	LogFlowFunc(("pVM=%p\n", pVM));
2601
2602	/*
2603	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2604	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2605	*/
2606	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2607	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2608	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2609	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2610	{
2611	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2612	return VERR_INTERNAL_ERROR;
2613	}
2614
2615	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2616	/*
2617	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2618	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2619	*/
2620	if ( pVM->hm.s.fAllow64BitGuests
2621	&& !HMVMX_IS_64BIT_HOST_MODE())
2622	{
2623	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2624	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2625	}
2626	#endif
2627
2628	/* Initialize these always, see hmR3InitFinalizeR0().*/
2629	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2630	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2631
2632	/* Setup the tagged-TLB flush handlers. */
2633	int rc = hmR0VmxSetupTaggedTlb(pVM);
2634	if (RT_FAILURE(rc))
2635	{
2636	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2637	return rc;
2638	}
2639
2640	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2641	{
2642	PVMCPU pVCpu = &pVM->aCpus[i];
2643	AssertPtr(pVCpu);
2644	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2645
2646	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2647	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2648
2649	/* Set revision dword at the beginning of the VMCS structure. */
2650	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2651
2652	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2653	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2654	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2655	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2656
2657	/* Load this VMCS as the current VMCS. */
2658	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2659	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2660	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2661
2662	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2663	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2664	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2665
2666	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2667	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2668	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2669
2670	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2671	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2672	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2673
2674	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2675	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2676	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2677
2678	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2679	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2680	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2681
2682	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2683	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2684	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2685	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2686	#endif
2687
2688	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2689	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2690	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2691	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2692
2693	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2694
2695	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2696	}
2697
2698	return VINF_SUCCESS;
2699	}
2700
2701
2702	/**
2703	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2704	* the VMCS.
2705	*
2706	* @returns VBox status code.
2707	* @param pVM Pointer to the VM.
2708	* @param pVCpu Pointer to the VMCPU.
2709	*/
2710	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2711	{
2712	NOREF(pVM); NOREF(pVCpu);
2713
2714	RTCCUINTREG uReg = ASMGetCR0();
2715	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2716	AssertRCReturn(rc, rc);
2717
2718	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2719	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2720	if (HMVMX_IS_64BIT_HOST_MODE())
2721	{
2722	uint64_t uRegCR3 = HMR0Get64bitCR3();
2723	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2724	}
2725	else
2726	#endif
2727	{
2728	uReg = ASMGetCR3();
2729	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2730	}
2731	AssertRCReturn(rc, rc);
2732
2733	uReg = ASMGetCR4();
2734	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2735	AssertRCReturn(rc, rc);
2736	return rc;
2737	}
2738
2739
2740	#if HC_ARCH_BITS == 64
2741	/**
2742	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2743	* requirements. See hmR0VmxSaveHostSegmentRegs().
2744	*/
2745	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2746	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2747	{ \
2748	bool fValidSelector = true; \
2749	if ((selValue) & X86_SEL_LDT) \
2750	{ \
2751	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2752	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2753	} \
2754	if (fValidSelector) \
2755	{ \
2756	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2757	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2758	} \
2759	(selValue) = 0; \
2760	}
2761	#endif
2762
2763
2764	/**
2765	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2766	* the host-state area in the VMCS.
2767	*
2768	* @returns VBox status code.
2769	* @param pVM Pointer to the VM.
2770	* @param pVCpu Pointer to the VMCPU.
2771	*/
2772	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2773	{
2774	NOREF(pVM);
2775	int rc = VERR_INTERNAL_ERROR_5;
2776
2777	/*
2778	* Host DS, ES, FS and GS segment registers.
2779	*/
2780	#if HC_ARCH_BITS == 64
2781	RTSEL uSelDS = ASMGetDS();
2782	RTSEL uSelES = ASMGetES();
2783	RTSEL uSelFS = ASMGetFS();
2784	RTSEL uSelGS = ASMGetGS();
2785	#else
2786	RTSEL uSelDS = 0;
2787	RTSEL uSelES = 0;
2788	RTSEL uSelFS = 0;
2789	RTSEL uSelGS = 0;
2790	#endif
2791
2792	/* Recalculate which host-state bits need to be manually restored. */
2793	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2794
2795	/*
2796	* Host CS and SS segment registers.
2797	*/
2798	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2799	RTSEL uSelCS;
2800	RTSEL uSelSS;
2801	if (HMVMX_IS_64BIT_HOST_MODE())
2802	{
2803	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2804	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2805	}
2806	else
2807	{
2808	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2809	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2810	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2811	}
2812	#else
2813	RTSEL uSelCS = ASMGetCS();
2814	RTSEL uSelSS = ASMGetSS();
2815	#endif
2816
2817	/*
2818	* Host TR segment register.
2819	*/
2820	RTSEL uSelTR = ASMGetTR();
2821
2822	#if HC_ARCH_BITS == 64
2823	/*
2824	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2825	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2826	*/
2827	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2828	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2829	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2830	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2831	# undef VMXLOCAL_ADJUST_HOST_SEG
2832	#endif
2833
2834	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2835	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2836	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2837	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2838	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2839	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2840	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2841	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2842	Assert(uSelCS);
2843	Assert(uSelTR);
2844
2845	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2846	#if 0
2847	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2848	Assert(uSelSS != 0);
2849	#endif
2850
2851	/* Write these host selector fields into the host-state area in the VMCS. */
2852	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2853	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2854	#if HC_ARCH_BITS == 64
2855	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2856	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2857	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2858	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2859	#endif
2860	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2861
2862	/*
2863	* Host GDTR and IDTR.
2864	*/
2865	RTGDTR Gdtr;
2866	RT_ZERO(Gdtr);
2867	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2868	if (HMVMX_IS_64BIT_HOST_MODE())
2869	{
2870	X86XDTR64 Gdtr64;
2871	X86XDTR64 Idtr64;
2872	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2873	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2874	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2875
2876	Gdtr.cbGdt = Gdtr64.cb;
2877	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2878	}
2879	else
2880	#endif
2881	{
2882	RTIDTR Idtr;
2883	ASMGetGDTR(&Gdtr);
2884	ASMGetIDTR(&Idtr);
2885	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2886	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2887
2888	#if HC_ARCH_BITS == 64
2889	/*
2890	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2891	* maximum limit (0xffff) on every VM-exit.
2892	*/
2893	if (Gdtr.cbGdt != 0xffff)
2894	{
2895	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2896	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2897	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2898	}
2899
2900	/*
2901	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2902	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2903	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2904	*/
2905	if (Idtr.cbIdt < 0x0fff)
2906	{
2907	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2908	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2909	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2910	}
2911	#endif
2912	}
2913
2914	/*
2915	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2916	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2917	*/
2918	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2919	{
2920	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2921	return VERR_VMX_INVALID_HOST_STATE;
2922	}
2923
2924	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2925	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2926	if (HMVMX_IS_64BIT_HOST_MODE())
2927	{
2928	/* We need the 64-bit TR base for hybrid darwin. */
2929	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2930	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2931	}
2932	else
2933	#endif
2934	{
2935	uintptr_t uTRBase;
2936	#if HC_ARCH_BITS == 64
2937	uTRBase = X86DESC64_BASE(pDesc);
2938
2939	/*
2940	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2941	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2942	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2943	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2944	*
2945	* [1] See Intel spec. 3.5 "System Descriptor Types".
2946	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2947	*/
2948	Assert(pDesc->System.u4Type == 11);
2949	if ( pDesc->System.u16LimitLow != 0x67
2950	\|\| pDesc->System.u4LimitHigh)
2951	{
2952	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2953	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2954
2955	/* Store the GDTR here as we need it while restoring TR. */
2956	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2957	}
2958	#else
2959	uTRBase = X86DESC_BASE(pDesc);
2960	#endif
2961	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2962	}
2963	AssertRCReturn(rc, rc);
2964
2965	/*
2966	* Host FS base and GS base.
2967	*/
2968	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2969	if (HMVMX_IS_64BIT_HOST_MODE())
2970	{
2971	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2972	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2973	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2974	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2975
2976	# if HC_ARCH_BITS == 64
2977	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2978	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2979	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2980	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2981	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2982	# endif
2983	}
2984	#endif
2985	return rc;
2986	}
2987
2988
2989	/**
2990	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2991	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2992	* the host after every successful VM-exit.
2993	*
2994	* @returns VBox status code.
2995	* @param pVM Pointer to the VM.
2996	* @param pVCpu Pointer to the VMCPU.
2997	*
2998	* @remarks No-long-jump zone!!!
2999	*/
3000	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3001	{
3002	NOREF(pVM);
3003
3004	AssertPtr(pVCpu);
3005	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3006
3007	int rc = VINF_SUCCESS;
3008	#if HC_ARCH_BITS == 64
3009	if (pVM->hm.s.fAllow64BitGuests)
3010	hmR0VmxLazySaveHostMsrs(pVCpu);
3011	#endif
3012
3013	if (pVCpu->hm.s.vmx.cMsrs > 0)
3014	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3015
3016	/*
3017	* Host Sysenter MSRs.
3018	*/
3019	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3020	AssertRCReturn(rc, rc);
3021	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3022	if (HMVMX_IS_64BIT_HOST_MODE())
3023	{
3024	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3025	AssertRCReturn(rc, rc);
3026	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3027	}
3028	else
3029	{
3030	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3031	AssertRCReturn(rc, rc);
3032	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3033	}
3034	#elif HC_ARCH_BITS == 32
3035	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3036	AssertRCReturn(rc, rc);
3037	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3038	#else
3039	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3040	AssertRCReturn(rc, rc);
3041	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3042	#endif
3043	AssertRCReturn(rc, rc);
3044
3045	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3046	* hmR0VmxSetupExitCtls() !! */
3047	return rc;
3048	}
3049
3050
3051	/**
3052	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3053	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3054	* controls".
3055	*
3056	* @returns VBox status code.
3057	* @param pVCpu Pointer to the VMCPU.
3058	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3059	* out-of-sync. Make sure to update the required fields
3060	* before using them.
3061	*
3062	* @remarks No-long-jump zone!!!
3063	*/
3064	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3065	{
3066	int rc = VINF_SUCCESS;
3067	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3068	{
3069	PVM pVM = pVCpu->CTX_SUFF(pVM);
3070	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3071	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3072
3073	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3074	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3075
3076	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3077	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3078	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3079	else
3080	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3081
3082	/*
3083	* The following should -not- be set (since we're not in SMM mode):
3084	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3085	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3086	*/
3087
3088	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3089	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3090	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3091
3092	if ((val & zap) != val)
3093	{
3094	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3095	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3096	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3097	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3098	}
3099
3100	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3101	AssertRCReturn(rc, rc);
3102
3103	/* Update VCPU with the currently set VM-exit controls. */
3104	pVCpu->hm.s.vmx.u32EntryCtls = val;
3105	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3106	}
3107	return rc;
3108	}
3109
3110
3111	/**
3112	* Sets up the VM-exit controls in the VMCS.
3113	*
3114	* @returns VBox status code.
3115	* @param pVM Pointer to the VM.
3116	* @param pVCpu Pointer to the VMCPU.
3117	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3118	* out-of-sync. Make sure to update the required fields
3119	* before using them.
3120	*
3121	* @remarks requires EFER.
3122	*/
3123	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3124	{
3125	NOREF(pMixedCtx);
3126
3127	int rc = VINF_SUCCESS;
3128	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3129	{
3130	PVM pVM = pVCpu->CTX_SUFF(pVM);
3131	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3132	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3133
3134	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3135	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3136
3137	/*
3138	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3139	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3140	*/
3141	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3142	if (HMVMX_IS_64BIT_HOST_MODE())
3143	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3144	else
3145	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3146	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3147	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3148	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3149	else
3150	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3151	#endif
3152
3153	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3154	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3155
3156	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3157	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3158	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3159	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3160	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3161
3162	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3163	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3164
3165	if ((val & zap) != val)
3166	{
3167	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3168	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3169	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3170	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3171	}
3172
3173	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3174	AssertRCReturn(rc, rc);
3175
3176	/* Update VCPU with the currently set VM-exit controls. */
3177	pVCpu->hm.s.vmx.u32ExitCtls = val;
3178	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3179	}
3180	return rc;
3181	}
3182
3183
3184	/**
3185	* Loads the guest APIC and related state.
3186	*
3187	* @returns VBox status code.
3188	* @param pVM Pointer to the VM.
3189	* @param pVCpu Pointer to the VMCPU.
3190	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3191	* out-of-sync. Make sure to update the required fields
3192	* before using them.
3193	*/
3194	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3195	{
3196	NOREF(pMixedCtx);
3197
3198	int rc = VINF_SUCCESS;
3199	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3200	{
3201	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3202	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3203	{
3204	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3205
3206	bool fPendingIntr = false;
3207	uint8_t u8Tpr = 0;
3208	uint8_t u8PendingIntr = 0;
3209	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3210	AssertRCReturn(rc, rc);
3211
3212	/*
3213	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3214	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3215	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3216	* the interrupt when we VM-exit for other reasons.
3217	*/
3218	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3219	uint32_t u32TprThreshold = 0;
3220	if (fPendingIntr)
3221	{
3222	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3223	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3224	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3225	if (u8PendingPriority <= u8TprPriority)
3226	u32TprThreshold = u8PendingPriority;
3227	else
3228	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3229	}
3230	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3231
3232	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3233	AssertRCReturn(rc, rc);
3234	}
3235
3236	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3237	}
3238	return rc;
3239	}
3240
3241
3242	/**
3243	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3244	*
3245	* @returns Guest's interruptibility-state.
3246	* @param pVCpu Pointer to the VMCPU.
3247	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3248	* out-of-sync. Make sure to update the required fields
3249	* before using them.
3250	*
3251	* @remarks No-long-jump zone!!!
3252	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3253	*/
3254	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3255	{
3256	/*
3257	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3258	* inhibit interrupts or clear any existing interrupt-inhibition.
3259	*/
3260	uint32_t uIntrState = 0;
3261	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3262	{
3263	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3264	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3265	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3266	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3267	{
3268	/*
3269	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3270	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3271	*/
3272	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3273	}
3274	else if (pMixedCtx->eflags.Bits.u1IF)
3275	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3276	else
3277	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3278	}
3279	return uIntrState;
3280	}
3281
3282
3283	/**
3284	* Loads the guest's interruptibility-state into the guest-state area in the
3285	* VMCS.
3286	*
3287	* @returns VBox status code.
3288	* @param pVCpu Pointer to the VMCPU.
3289	* @param uIntrState The interruptibility-state to set.
3290	*/
3291	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3292	{
3293	NOREF(pVCpu);
3294	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3295	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3296	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3297	AssertRCReturn(rc, rc);
3298	return rc;
3299	}
3300
3301
3302	/**
3303	* Loads the guest's RIP into the guest-state area in the VMCS.
3304	*
3305	* @returns VBox status code.
3306	* @param pVCpu Pointer to the VMCPU.
3307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3308	* out-of-sync. Make sure to update the required fields
3309	* before using them.
3310	*
3311	* @remarks No-long-jump zone!!!
3312	*/
3313	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3314	{
3315	int rc = VINF_SUCCESS;
3316	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3317	{
3318	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3319	AssertRCReturn(rc, rc);
3320
3321	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3322	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3323	}
3324	return rc;
3325	}
3326
3327
3328	/**
3329	* Loads the guest's RSP into the guest-state area in the VMCS.
3330	*
3331	* @returns VBox status code.
3332	* @param pVCpu Pointer to the VMCPU.
3333	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3334	* out-of-sync. Make sure to update the required fields
3335	* before using them.
3336	*
3337	* @remarks No-long-jump zone!!!
3338	*/
3339	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3340	{
3341	int rc = VINF_SUCCESS;
3342	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3343	{
3344	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3345	AssertRCReturn(rc, rc);
3346
3347	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3348	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3349	}
3350	return rc;
3351	}
3352
3353
3354	/**
3355	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3356	*
3357	* @returns VBox status code.
3358	* @param pVCpu Pointer to the VMCPU.
3359	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3360	* out-of-sync. Make sure to update the required fields
3361	* before using them.
3362	*
3363	* @remarks No-long-jump zone!!!
3364	*/
3365	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3366	{
3367	int rc = VINF_SUCCESS;
3368	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3369	{
3370	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3371	Let us assert it as such and use 32-bit VMWRITE. */
3372	Assert(!(pMixedCtx->rflags.u64 >> 32));
3373	X86EFLAGS Eflags = pMixedCtx->eflags;
3374	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3375	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3376
3377	/*
3378	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3379	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3380	*/
3381	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3382	{
3383	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3384	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3385	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3386	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3387	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3388	}
3389
3390	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3391	AssertRCReturn(rc, rc);
3392
3393	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3394	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3395	}
3396	return rc;
3397	}
3398
3399
3400	/**
3401	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3402	*
3403	* @returns VBox status code.
3404	* @param pVCpu Pointer to the VMCPU.
3405	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3406	* out-of-sync. Make sure to update the required fields
3407	* before using them.
3408	*
3409	* @remarks No-long-jump zone!!!
3410	*/
3411	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3412	{
3413	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3414	AssertRCReturn(rc, rc);
3415	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3416	AssertRCReturn(rc, rc);
3417	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3418	AssertRCReturn(rc, rc);
3419	return rc;
3420	}
3421
3422
3423	/**
3424	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3425	* CR0 is partially shared with the host and we have to consider the FPU bits.
3426	*
3427	* @returns VBox status code.
3428	* @param pVM Pointer to the VM.
3429	* @param pVCpu Pointer to the VMCPU.
3430	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3431	* out-of-sync. Make sure to update the required fields
3432	* before using them.
3433	*
3434	* @remarks No-long-jump zone!!!
3435	*/
3436	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3437	{
3438	/*
3439	* Guest CR0.
3440	* Guest FPU.
3441	*/
3442	int rc = VINF_SUCCESS;
3443	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3444	{
3445	Assert(!(pMixedCtx->cr0 >> 32));
3446	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3447	PVM pVM = pVCpu->CTX_SUFF(pVM);
3448
3449	/* The guest's view (read access) of its CR0 is unblemished. */
3450	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3451	AssertRCReturn(rc, rc);
3452	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3453
3454	/* Setup VT-x's view of the guest CR0. */
3455	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3456	if (pVM->hm.s.fNestedPaging)
3457	{
3458	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3459	{
3460	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3461	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3462	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3463	}
3464	else
3465	{
3466	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3467	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3468	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3469	}
3470
3471	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3472	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3473	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3474
3475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3476	AssertRCReturn(rc, rc);
3477	}
3478	else
3479	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3480
3481	/*
3482	* Guest FPU bits.
3483	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3484	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3485	*/
3486	u32GuestCR0 \|= X86_CR0_NE;
3487	bool fInterceptNM = false;
3488	if (CPUMIsGuestFPUStateActive(pVCpu))
3489	{
3490	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3491	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3492	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3493	}
3494	else
3495	{
3496	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3497	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3498	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3499	}
3500
3501	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3502	bool fInterceptMF = false;
3503	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3504	fInterceptMF = true;
3505
3506	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3507	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3508	{
3509	Assert(PDMVmmDevHeapIsEnabled(pVM));
3510	Assert(pVM->hm.s.vmx.pRealModeTSS);
3511	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3512	fInterceptNM = true;
3513	fInterceptMF = true;
3514	}
3515	else
3516	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3517
3518	if (fInterceptNM)
3519	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3520	else
3521	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3522
3523	if (fInterceptMF)
3524	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3525	else
3526	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3527
3528	/* Additional intercepts for debugging, define these yourself explicitly. */
3529	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3530	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3531	\| RT_BIT(X86_XCPT_BP)
3532	\| RT_BIT(X86_XCPT_DB)
3533	\| RT_BIT(X86_XCPT_DE)
3534	\| RT_BIT(X86_XCPT_NM)
3535	\| RT_BIT(X86_XCPT_UD)
3536	\| RT_BIT(X86_XCPT_NP)
3537	\| RT_BIT(X86_XCPT_SS)
3538	\| RT_BIT(X86_XCPT_GP)
3539	\| RT_BIT(X86_XCPT_PF)
3540	\| RT_BIT(X86_XCPT_MF)
3541	;
3542	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3543	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3544	#endif
3545
3546	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3547
3548	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3549	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3550	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3551	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3552	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3553	else
3554	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3555
3556	u32GuestCR0 \|= uSetCR0;
3557	u32GuestCR0 &= uZapCR0;
3558	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3559
3560	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3561	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3562	AssertRCReturn(rc, rc);
3563	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3564	AssertRCReturn(rc, rc);
3565	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3566
3567	/*
3568	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3569	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3570	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3571	*/
3572	uint32_t u32CR0Mask = 0;
3573	u32CR0Mask = X86_CR0_PE
3574	\| X86_CR0_NE
3575	\| X86_CR0_WP
3576	\| X86_CR0_PG
3577	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3578	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3579	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3580
3581	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3582	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3583	* and @bugref{6944}. */
3584	#if 0
3585	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3586	u32CR0Mask &= ~X86_CR0_PE;
3587	#endif
3588	if (pVM->hm.s.fNestedPaging)
3589	u32CR0Mask &= ~X86_CR0_WP;
3590
3591	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3592	if (fInterceptNM)
3593	{
3594	u32CR0Mask \|= X86_CR0_TS
3595	\| X86_CR0_MP;
3596	}
3597
3598	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3599	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3600	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3601	AssertRCReturn(rc, rc);
3602	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3603
3604	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3605	}
3606	return rc;
3607	}
3608
3609
3610	/**
3611	* Loads the guest control registers (CR3, CR4) into the guest-state area
3612	* in the VMCS.
3613	*
3614	* @returns VBox status code.
3615	* @param pVM Pointer to the VM.
3616	* @param pVCpu Pointer to the VMCPU.
3617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3618	* out-of-sync. Make sure to update the required fields
3619	* before using them.
3620	*
3621	* @remarks No-long-jump zone!!!
3622	*/
3623	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3624	{
3625	int rc = VINF_SUCCESS;
3626	PVM pVM = pVCpu->CTX_SUFF(pVM);
3627
3628	/*
3629	* Guest CR2.
3630	* It's always loaded in the assembler code. Nothing to do here.
3631	*/
3632
3633	/*
3634	* Guest CR3.
3635	*/
3636	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3637	{
3638	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3639	if (pVM->hm.s.fNestedPaging)
3640	{
3641	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3642
3643	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3644	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3645	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3646	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3647
3648	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3649	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3650	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3651
3652	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3653	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3654	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3655	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3656
3657	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3658	AssertRCReturn(rc, rc);
3659	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3660
3661	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3662	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3663	{
3664	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3665	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3666	{
3667	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3668	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3669	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3670	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3671	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3672	}
3673
3674	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3675	have Unrestricted Execution to handle the guest when it's not using paging. */
3676	GCPhysGuestCR3 = pMixedCtx->cr3;
3677	}
3678	else
3679	{
3680	/*
3681	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3682	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3683	* EPT takes care of translating it to host-physical addresses.
3684	*/
3685	RTGCPHYS GCPhys;
3686	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3687	Assert(PDMVmmDevHeapIsEnabled(pVM));
3688
3689	/* We obtain it here every time as the guest could have relocated this PCI region. */
3690	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3691	AssertRCReturn(rc, rc);
3692
3693	GCPhysGuestCR3 = GCPhys;
3694	}
3695
3696	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3697	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3698	}
3699	else
3700	{
3701	/* Non-nested paging case, just use the hypervisor's CR3. */
3702	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3703
3704	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3705	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3706	}
3707	AssertRCReturn(rc, rc);
3708
3709	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3710	}
3711
3712	/*
3713	* Guest CR4.
3714	*/
3715	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3716	{
3717	Assert(!(pMixedCtx->cr4 >> 32));
3718	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3719
3720	/* The guest's view of its CR4 is unblemished. */
3721	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3722	AssertRCReturn(rc, rc);
3723	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3724
3725	/* Setup VT-x's view of the guest CR4. */
3726	/*
3727	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3728	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3729	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3730	*/
3731	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3732	{
3733	Assert(pVM->hm.s.vmx.pRealModeTSS);
3734	Assert(PDMVmmDevHeapIsEnabled(pVM));
3735	u32GuestCR4 &= ~X86_CR4_VME;
3736	}
3737
3738	if (pVM->hm.s.fNestedPaging)
3739	{
3740	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3741	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3742	{
3743	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3744	u32GuestCR4 \|= X86_CR4_PSE;
3745	/* Our identity mapping is a 32-bit page directory. */
3746	u32GuestCR4 &= ~X86_CR4_PAE;
3747	}
3748	/* else use guest CR4.*/
3749	}
3750	else
3751	{
3752	/*
3753	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3754	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3755	*/
3756	switch (pVCpu->hm.s.enmShadowMode)
3757	{
3758	case PGMMODE_REAL: /* Real-mode. */
3759	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3760	case PGMMODE_32_BIT: /* 32-bit paging. */
3761	{
3762	u32GuestCR4 &= ~X86_CR4_PAE;
3763	break;
3764	}
3765
3766	case PGMMODE_PAE: /* PAE paging. */
3767	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3768	{
3769	u32GuestCR4 \|= X86_CR4_PAE;
3770	break;
3771	}
3772
3773	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3774	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3775	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3776	break;
3777	#endif
3778	default:
3779	AssertFailed();
3780	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3781	}
3782	}
3783
3784	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3785	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3786	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3787	u32GuestCR4 \|= uSetCR4;
3788	u32GuestCR4 &= uZapCR4;
3789
3790	/* Write VT-x's view of the guest CR4 into the VMCS. */
3791	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3792	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3793	AssertRCReturn(rc, rc);
3794
3795	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3796	uint32_t u32CR4Mask = 0;
3797	u32CR4Mask = X86_CR4_VME
3798	\| X86_CR4_PAE
3799	\| X86_CR4_PGE
3800	\| X86_CR4_PSE
3801	\| X86_CR4_VMXE;
3802	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3803	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3804	AssertRCReturn(rc, rc);
3805
3806	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3807	}
3808	return rc;
3809	}
3810
3811
3812	/**
3813	* Loads the guest debug registers into the guest-state area in the VMCS.
3814	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3815	*
3816	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3817	*
3818	* @returns VBox status code.
3819	* @param pVCpu Pointer to the VMCPU.
3820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3821	* out-of-sync. Make sure to update the required fields
3822	* before using them.
3823	*
3824	* @remarks No-long-jump zone!!!
3825	*/
3826	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3827	{
3828	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3829	return VINF_SUCCESS;
3830
3831	#ifdef VBOX_STRICT
3832	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3833	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3834	{
3835	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3836	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3837	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3838	}
3839	#endif
3840
3841	int rc;
3842	PVM pVM = pVCpu->CTX_SUFF(pVM);
3843	bool fInterceptDB = false;
3844	bool fInterceptMovDRx = false;
3845	if ( pVCpu->hm.s.fSingleInstruction
3846	\|\| DBGFIsStepping(pVCpu))
3847	{
3848	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3849	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3850	{
3851	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3852	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3853	AssertRCReturn(rc, rc);
3854	Assert(fInterceptDB == false);
3855	}
3856	else
3857	{
3858	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3859	pVCpu->hm.s.fClearTrapFlag = true;
3860	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3861	fInterceptDB = true;
3862	}
3863	}
3864
3865	if ( fInterceptDB
3866	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3867	{
3868	/*
3869	* Use the combined guest and host DRx values found in the hypervisor
3870	* register set because the debugger has breakpoints active or someone
3871	* is single stepping on the host side without a monitor trap flag.
3872	*
3873	* Note! DBGF expects a clean DR6 state before executing guest code.
3874	*/
3875	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3876	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3877	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3878	{
3879	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3880	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3881	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3882	}
3883	else
3884	#endif
3885	if (!CPUMIsHyperDebugStateActive(pVCpu))
3886	{
3887	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3888	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3889	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3890	}
3891
3892	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3893	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3894	AssertRCReturn(rc, rc);
3895
3896	pVCpu->hm.s.fUsingHyperDR7 = true;
3897	fInterceptDB = true;
3898	fInterceptMovDRx = true;
3899	}
3900	else
3901	{
3902	/*
3903	* If the guest has enabled debug registers, we need to load them prior to
3904	* executing guest code so they'll trigger at the right time.
3905	*/
3906	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3907	{
3908	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3909	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3910	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3911	{
3912	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3913	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3914	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3915	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3916	}
3917	else
3918	#endif
3919	if (!CPUMIsGuestDebugStateActive(pVCpu))
3920	{
3921	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3922	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3923	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3924	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3925	}
3926	Assert(!fInterceptDB);
3927	Assert(!fInterceptMovDRx);
3928	}
3929	/*
3930	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3931	* must intercept #DB in order to maintain a correct DR6 guest value.
3932	*/
3933	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3934	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3935	&& !CPUMIsGuestDebugStateActive(pVCpu))
3936	#else
3937	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3938	#endif
3939	{
3940	fInterceptMovDRx = true;
3941	fInterceptDB = true;
3942	}
3943
3944	/* Update guest DR7. */
3945	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3946	AssertRCReturn(rc, rc);
3947
3948	pVCpu->hm.s.fUsingHyperDR7 = false;
3949	}
3950
3951	/*
3952	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3953	*/
3954	if (fInterceptDB)
3955	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3956	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3957	{
3958	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3959	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3960	#endif
3961	}
3962	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3963	AssertRCReturn(rc, rc);
3964
3965	/*
3966	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3967	*/
3968	if (fInterceptMovDRx)
3969	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3970	else
3971	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3972	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3973	AssertRCReturn(rc, rc);
3974
3975	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3976	return VINF_SUCCESS;
3977	}
3978
3979
3980	#ifdef VBOX_STRICT
3981	/**
3982	* Strict function to validate segment registers.
3983	*
3984	* @remarks ASSUMES CR0 is up to date.
3985	*/
3986	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3987	{
3988	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3989	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3990	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3991	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3992	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3993	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3994	{
3995	/* Protected mode checks */
3996	/* CS */
3997	Assert(pCtx->cs.Attr.n.u1Present);
3998	Assert(!(pCtx->cs.Attr.u & 0xf00));
3999	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4000	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4001	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4002	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4003	\|\| (pCtx->cs.Attr.n.u1Granularity));
4004	/* CS cannot be loaded with NULL in protected mode. */
4005	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4006	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4007	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4008	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4009	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4010	else
4011	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4012	/* SS */
4013	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4014	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4015	if ( !(pCtx->cr0 & X86_CR0_PE)
4016	\|\| pCtx->cs.Attr.n.u4Type == 3)
4017	{
4018	Assert(!pCtx->ss.Attr.n.u2Dpl);
4019	}
4020	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4021	{
4022	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4023	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4024	Assert(pCtx->ss.Attr.n.u1Present);
4025	Assert(!(pCtx->ss.Attr.u & 0xf00));
4026	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4027	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4028	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4029	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4030	\|\| (pCtx->ss.Attr.n.u1Granularity));
4031	}
4032	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4033	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4034	{
4035	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4036	Assert(pCtx->ds.Attr.n.u1Present);
4037	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4038	Assert(!(pCtx->ds.Attr.u & 0xf00));
4039	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4040	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4041	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4042	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4043	\|\| (pCtx->ds.Attr.n.u1Granularity));
4044	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4045	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4046	}
4047	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4048	{
4049	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4050	Assert(pCtx->es.Attr.n.u1Present);
4051	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4052	Assert(!(pCtx->es.Attr.u & 0xf00));
4053	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4054	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4055	\|\| !(pCtx->es.Attr.n.u1Granularity));
4056	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4057	\|\| (pCtx->es.Attr.n.u1Granularity));
4058	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4059	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4060	}
4061	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4062	{
4063	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4064	Assert(pCtx->fs.Attr.n.u1Present);
4065	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4066	Assert(!(pCtx->fs.Attr.u & 0xf00));
4067	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4068	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4069	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4070	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4071	\|\| (pCtx->fs.Attr.n.u1Granularity));
4072	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4073	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4074	}
4075	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4076	{
4077	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4078	Assert(pCtx->gs.Attr.n.u1Present);
4079	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4080	Assert(!(pCtx->gs.Attr.u & 0xf00));
4081	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4082	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4083	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4084	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4085	\|\| (pCtx->gs.Attr.n.u1Granularity));
4086	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4087	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4088	}
4089	/* 64-bit capable CPUs. */
4090	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4091	if (HMVMX_IS_64BIT_HOST_MODE())
4092	{
4093	Assert(!(pCtx->cs.u64Base >> 32));
4094	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4095	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4096	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4097	}
4098	# endif
4099	}
4100	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4101	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4102	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4103	{
4104	/* Real and v86 mode checks. */
4105	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4106	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4107	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4108	{
4109	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4110	}
4111	else
4112	{
4113	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4114	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4115	}
4116
4117	/* CS */
4118	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4119	Assert(pCtx->cs.u32Limit == 0xffff);
4120	Assert(u32CSAttr == 0xf3);
4121	/* SS */
4122	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4123	Assert(pCtx->ss.u32Limit == 0xffff);
4124	Assert(u32SSAttr == 0xf3);
4125	/* DS */
4126	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4127	Assert(pCtx->ds.u32Limit == 0xffff);
4128	Assert(u32DSAttr == 0xf3);
4129	/* ES */
4130	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4131	Assert(pCtx->es.u32Limit == 0xffff);
4132	Assert(u32ESAttr == 0xf3);
4133	/* FS */
4134	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4135	Assert(pCtx->fs.u32Limit == 0xffff);
4136	Assert(u32FSAttr == 0xf3);
4137	/* GS */
4138	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4139	Assert(pCtx->gs.u32Limit == 0xffff);
4140	Assert(u32GSAttr == 0xf3);
4141	/* 64-bit capable CPUs. */
4142	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4143	if (HMVMX_IS_64BIT_HOST_MODE())
4144	{
4145	Assert(!(pCtx->cs.u64Base >> 32));
4146	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4147	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4148	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4149	}
4150	# endif
4151	}
4152	}
4153	#endif /* VBOX_STRICT */
4154
4155
4156	/**
4157	* Writes a guest segment register into the guest-state area in the VMCS.
4158	*
4159	* @returns VBox status code.
4160	* @param pVCpu Pointer to the VMCPU.
4161	* @param idxSel Index of the selector in the VMCS.
4162	* @param idxLimit Index of the segment limit in the VMCS.
4163	* @param idxBase Index of the segment base in the VMCS.
4164	* @param idxAccess Index of the access rights of the segment in the VMCS.
4165	* @param pSelReg Pointer to the segment selector.
4166	*
4167	* @remarks No-long-jump zone!!!
4168	*/
4169	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4170	uint32_t idxAccess, PCPUMSELREG pSelReg)
4171	{
4172	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4173	AssertRCReturn(rc, rc);
4174	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4175	AssertRCReturn(rc, rc);
4176	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4177	AssertRCReturn(rc, rc);
4178
4179	uint32_t u32Access = pSelReg->Attr.u;
4180	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4181	{
4182	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4183	u32Access = 0xf3;
4184	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4185	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4186	}
4187	else
4188	{
4189	/*
4190	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4191	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4192	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4193	* loaded in protected-mode have their attribute as 0.
4194	*/
4195	if (!u32Access)
4196	u32Access = X86DESCATTR_UNUSABLE;
4197	}
4198
4199	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4200	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4201	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4202
4203	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4204	AssertRCReturn(rc, rc);
4205	return rc;
4206	}
4207
4208
4209	/**
4210	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4211	* into the guest-state area in the VMCS.
4212	*
4213	* @returns VBox status code.
4214	* @param pVM Pointer to the VM.
4215	* @param pVCPU Pointer to the VMCPU.
4216	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4217	* out-of-sync. Make sure to update the required fields
4218	* before using them.
4219	*
4220	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4221	* @remarks No-long-jump zone!!!
4222	*/
4223	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4224	{
4225	int rc = VERR_INTERNAL_ERROR_5;
4226	PVM pVM = pVCpu->CTX_SUFF(pVM);
4227
4228	/*
4229	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4230	*/
4231	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4232	{
4233	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4234	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4235	{
4236	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4237	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4238	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4239	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4240	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4241	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4242	}
4243
4244	#ifdef VBOX_WITH_REM
4245	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4246	{
4247	Assert(pVM->hm.s.vmx.pRealModeTSS);
4248	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4249	if ( pVCpu->hm.s.vmx.fWasInRealMode
4250	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4251	{
4252	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4253	in real-mode (e.g. OpenBSD 4.0) */
4254	REMFlushTBs(pVM);
4255	Log4(("Load: Switch to protected mode detected!\n"));
4256	pVCpu->hm.s.vmx.fWasInRealMode = false;
4257	}
4258	}
4259	#endif
4260	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4261	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4262	AssertRCReturn(rc, rc);
4263	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4264	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4265	AssertRCReturn(rc, rc);
4266	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4267	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4268	AssertRCReturn(rc, rc);
4269	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4270	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4271	AssertRCReturn(rc, rc);
4272	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4273	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4274	AssertRCReturn(rc, rc);
4275	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4276	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4277	AssertRCReturn(rc, rc);
4278
4279	#ifdef VBOX_STRICT
4280	/* Validate. */
4281	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4282	#endif
4283
4284	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4285	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4286	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4287	}
4288
4289	/*
4290	* Guest TR.
4291	*/
4292	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4293	{
4294	/*
4295	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4296	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4297	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4298	*/
4299	uint16_t u16Sel = 0;
4300	uint32_t u32Limit = 0;
4301	uint64_t u64Base = 0;
4302	uint32_t u32AccessRights = 0;
4303
4304	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4305	{
4306	u16Sel = pMixedCtx->tr.Sel;
4307	u32Limit = pMixedCtx->tr.u32Limit;
4308	u64Base = pMixedCtx->tr.u64Base;
4309	u32AccessRights = pMixedCtx->tr.Attr.u;
4310	}
4311	else
4312	{
4313	Assert(pVM->hm.s.vmx.pRealModeTSS);
4314	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4315
4316	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4317	RTGCPHYS GCPhys;
4318	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4319	AssertRCReturn(rc, rc);
4320
4321	X86DESCATTR DescAttr;
4322	DescAttr.u = 0;
4323	DescAttr.n.u1Present = 1;
4324	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4325
4326	u16Sel = 0;
4327	u32Limit = HM_VTX_TSS_SIZE;
4328	u64Base = GCPhys; /* in real-mode phys = virt. */
4329	u32AccessRights = DescAttr.u;
4330	}
4331
4332	/* Validate. */
4333	Assert(!(u16Sel & RT_BIT(2)));
4334	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4335	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4336	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4337	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4338	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4339	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4340	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4341	Assert( (u32Limit & 0xfff) == 0xfff
4342	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4343	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4344	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4345
4346	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4347	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4348	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4349	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4350
4351	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4352	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4353	}
4354
4355	/*
4356	* Guest GDTR.
4357	*/
4358	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4359	{
4360	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4361	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4362
4363	/* Validate. */
4364	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4365
4366	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4367	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4368	}
4369
4370	/*
4371	* Guest LDTR.
4372	*/
4373	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4374	{
4375	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4376	uint32_t u32Access = 0;
4377	if (!pMixedCtx->ldtr.Attr.u)
4378	u32Access = X86DESCATTR_UNUSABLE;
4379	else
4380	u32Access = pMixedCtx->ldtr.Attr.u;
4381
4382	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4383	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4384	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4385	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4386
4387	/* Validate. */
4388	if (!(u32Access & X86DESCATTR_UNUSABLE))
4389	{
4390	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4391	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4392	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4393	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4394	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4395	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4396	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4397	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4398	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4399	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4400	}
4401
4402	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4403	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4404	}
4405
4406	/*
4407	* Guest IDTR.
4408	*/
4409	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4410	{
4411	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4412	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4413
4414	/* Validate. */
4415	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4416
4417	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4418	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4419	}
4420
4421	return VINF_SUCCESS;
4422	}
4423
4424
4425	/**
4426	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4427	* areas. These MSRs will automatically be loaded to the host CPU on every
4428	* successful VM entry and stored from the host CPU on every successful VM-exit.
4429	*
4430	* This also creates/updates MSR slots for the host MSRs. The actual host
4431	* MSR values are -not- updated here for performance reasons. See
4432	* hmR0VmxSaveHostMsrs().
4433	*
4434	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4435	*
4436	* @returns VBox status code.
4437	* @param pVCpu Pointer to the VMCPU.
4438	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4439	* out-of-sync. Make sure to update the required fields
4440	* before using them.
4441	*
4442	* @remarks No-long-jump zone!!!
4443	*/
4444	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4445	{
4446	AssertPtr(pVCpu);
4447	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4448
4449	/*
4450	* MSRs that we use the auto-load/store MSR area in the VMCS.
4451	*/
4452	PVM pVM = pVCpu->CTX_SUFF(pVM);
4453	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4454	{
4455	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4456	if (pVM->hm.s.fAllow64BitGuests)
4457	{
4458	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4459	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4460	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4461	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4462	# ifdef DEBUG
4463	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4464	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4465	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4466	# endif
4467	}
4468	#endif
4469	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4470	}
4471
4472	/*
4473	* Guest Sysenter MSRs.
4474	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4475	* VM-exits on WRMSRs for these MSRs.
4476	*/
4477	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4478	{
4479	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4480	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4481	}
4482
4483	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4484	{
4485	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4486	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4487	}
4488
4489	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4490	{
4491	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4492	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4493	}
4494
4495	return VINF_SUCCESS;
4496	}
4497
4498
4499	/**
4500	* Loads the guest activity state into the guest-state area in the VMCS.
4501	*
4502	* @returns VBox status code.
4503	* @param pVCpu Pointer to the VMCPU.
4504	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4505	* out-of-sync. Make sure to update the required fields
4506	* before using them.
4507	*
4508	* @remarks No-long-jump zone!!!
4509	*/
4510	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4511	{
4512	NOREF(pCtx);
4513	/** @todo See if we can make use of other states, e.g.
4514	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4515	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4516	{
4517	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4518	AssertRCReturn(rc, rc);
4519
4520	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4521	}
4522	return VINF_SUCCESS;
4523	}
4524
4525
4526	/**
4527	* Sets up the appropriate function to run guest code.
4528	*
4529	* @returns VBox status code.
4530	* @param pVCpu Pointer to the VMCPU.
4531	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4532	* out-of-sync. Make sure to update the required fields
4533	* before using them.
4534	*
4535	* @remarks No-long-jump zone!!!
4536	*/
4537	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4538	{
4539	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4540	{
4541	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4542	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4543	#endif
4544	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4545	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4546	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4547	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4548	{
4549	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4550	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4551	}
4552	#else
4553	/* 64-bit host or hybrid host. */
4554	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4555	#endif
4556	}
4557	else
4558	{
4559	/* Guest is not in long mode, use the 32-bit handler. */
4560	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4561	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4562	{
4563	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4564	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4565	}
4566	#else
4567	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4568	#endif
4569	}
4570	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4571	return VINF_SUCCESS;
4572	}
4573
4574
4575	/**
4576	* Wrapper for running the guest code in VT-x.
4577	*
4578	* @returns VBox strict status code.
4579	* @param pVM Pointer to the VM.
4580	* @param pVCpu Pointer to the VMCPU.
4581	* @param pCtx Pointer to the guest-CPU context.
4582	*
4583	* @remarks No-long-jump zone!!!
4584	*/
4585	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4586	{
4587	/*
4588	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4589	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4590	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4591	*/
4592	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4593	/** @todo Add stats for resume vs launch. */
4594	#ifdef VBOX_WITH_KERNEL_USING_XMM
4595	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4596	#else
4597	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4598	#endif
4599	}
4600
4601
4602	/**
4603	* Reports world-switch error and dumps some useful debug info.
4604	*
4605	* @param pVM Pointer to the VM.
4606	* @param pVCpu Pointer to the VMCPU.
4607	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4608	* @param pCtx Pointer to the guest-CPU context.
4609	* @param pVmxTransient Pointer to the VMX transient structure (only
4610	* exitReason updated).
4611	*/
4612	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4613	{
4614	Assert(pVM);
4615	Assert(pVCpu);
4616	Assert(pCtx);
4617	Assert(pVmxTransient);
4618	HMVMX_ASSERT_PREEMPT_SAFE();
4619
4620	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4621	switch (rcVMRun)
4622	{
4623	case VERR_VMX_INVALID_VMXON_PTR:
4624	AssertFailed();
4625	break;
4626	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4627	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4628	{
4629	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4630	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4631	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4632	AssertRC(rc);
4633
4634	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4635	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4636	Cannot do it here as we may have been long preempted. */
4637
4638	#ifdef VBOX_STRICT
4639	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4640	pVmxTransient->uExitReason));
4641	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4642	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4643	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4644	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4645	else
4646	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4647	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4648	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4649
4650	/* VMX control bits. */
4651	uint32_t u32Val;
4652	uint64_t u64Val;
4653	HMVMXHCUINTREG uHCReg;
4654	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4655	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4656	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4657	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4658	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4659	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4660	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4661	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4663	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4664	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4665	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4667	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4668	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4669	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4671	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4672	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4673	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4675	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4677	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4678	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4679	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4680	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4681	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4683	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4685	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4686	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4687	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4688	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4689	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4690	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4691	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4692	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4693	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4694	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4695	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4696
4697	/* Guest bits. */
4698	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4699	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4700	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4701	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4702	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4703	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4704	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4705	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4706
4707	/* Host bits. */
4708	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4709	Log4(("Host CR0 %#RHr\n", uHCReg));
4710	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4711	Log4(("Host CR3 %#RHr\n", uHCReg));
4712	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4713	Log4(("Host CR4 %#RHr\n", uHCReg));
4714
4715	RTGDTR HostGdtr;
4716	PCX86DESCHC pDesc;
4717	ASMGetGDTR(&HostGdtr);
4718	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4719	Log4(("Host CS %#08x\n", u32Val));
4720	if (u32Val < HostGdtr.cbGdt)
4721	{
4722	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4723	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4724	}
4725
4726	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4727	Log4(("Host DS %#08x\n", u32Val));
4728	if (u32Val < HostGdtr.cbGdt)
4729	{
4730	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4731	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4732	}
4733
4734	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4735	Log4(("Host ES %#08x\n", u32Val));
4736	if (u32Val < HostGdtr.cbGdt)
4737	{
4738	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4739	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4740	}
4741
4742	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4743	Log4(("Host FS %#08x\n", u32Val));
4744	if (u32Val < HostGdtr.cbGdt)
4745	{
4746	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4747	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4748	}
4749
4750	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4751	Log4(("Host GS %#08x\n", u32Val));
4752	if (u32Val < HostGdtr.cbGdt)
4753	{
4754	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4755	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4756	}
4757
4758	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4759	Log4(("Host SS %#08x\n", u32Val));
4760	if (u32Val < HostGdtr.cbGdt)
4761	{
4762	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4763	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4764	}
4765
4766	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4767	Log4(("Host TR %#08x\n", u32Val));
4768	if (u32Val < HostGdtr.cbGdt)
4769	{
4770	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4771	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4772	}
4773
4774	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4775	Log4(("Host TR Base %#RHv\n", uHCReg));
4776	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4777	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4778	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4779	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4780	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4781	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4782	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4783	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4784	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4785	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4786	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4787	Log4(("Host RSP %#RHv\n", uHCReg));
4788	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4789	Log4(("Host RIP %#RHv\n", uHCReg));
4790	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4791	if (HMVMX_IS_64BIT_HOST_MODE())
4792	{
4793	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4794	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4795	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4796	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4797	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4798	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4799	}
4800	# endif
4801	#endif /* VBOX_STRICT */
4802	break;
4803	}
4804
4805	default:
4806	/* Impossible */
4807	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4808	break;
4809	}
4810	NOREF(pVM); NOREF(pCtx);
4811	}
4812
4813
4814	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4815	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4816	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4817	#endif
4818	#ifdef VBOX_STRICT
4819	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4820	{
4821	switch (idxField)
4822	{
4823	case VMX_VMCS_GUEST_RIP:
4824	case VMX_VMCS_GUEST_RSP:
4825	case VMX_VMCS_GUEST_SYSENTER_EIP:
4826	case VMX_VMCS_GUEST_SYSENTER_ESP:
4827	case VMX_VMCS_GUEST_GDTR_BASE:
4828	case VMX_VMCS_GUEST_IDTR_BASE:
4829	case VMX_VMCS_GUEST_CS_BASE:
4830	case VMX_VMCS_GUEST_DS_BASE:
4831	case VMX_VMCS_GUEST_ES_BASE:
4832	case VMX_VMCS_GUEST_FS_BASE:
4833	case VMX_VMCS_GUEST_GS_BASE:
4834	case VMX_VMCS_GUEST_SS_BASE:
4835	case VMX_VMCS_GUEST_LDTR_BASE:
4836	case VMX_VMCS_GUEST_TR_BASE:
4837	case VMX_VMCS_GUEST_CR3:
4838	return true;
4839	}
4840	return false;
4841	}
4842
4843	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4844	{
4845	switch (idxField)
4846	{
4847	/* Read-only fields. */
4848	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4849	return true;
4850	}
4851	/* Remaining readable fields should also be writable. */
4852	return hmR0VmxIsValidWriteField(idxField);
4853	}
4854	#endif /* VBOX_STRICT */
4855
4856
4857	/**
4858	* Executes the specified handler in 64-bit mode.
4859	*
4860	* @returns VBox status code.
4861	* @param pVM Pointer to the VM.
4862	* @param pVCpu Pointer to the VMCPU.
4863	* @param pCtx Pointer to the guest CPU context.
4864	* @param enmOp The operation to perform.
4865	* @param cbParam Number of parameters.
4866	* @param paParam Array of 32-bit parameters.
4867	*/
4868	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4869	uint32_t *paParam)
4870	{
4871	int rc, rc2;
4872	PHMGLOBALCPUINFO pCpu;
4873	RTHCPHYS HCPhysCpuPage;
4874	RTCCUINTREG uOldEflags;
4875
4876	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4877	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4878	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4879	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4880
4881	#ifdef VBOX_STRICT
4882	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4883	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4884
4885	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4886	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4887	#endif
4888
4889	/* Disable interrupts. */
4890	uOldEflags = ASMIntDisableFlags();
4891
4892	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4893	RTCPUID idHostCpu = RTMpCpuId();
4894	CPUMR0SetLApic(pVCpu, idHostCpu);
4895	#endif
4896
4897	pCpu = HMR0GetCurrentCpu();
4898	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4899
4900	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4901	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4902
4903	/* Leave VMX Root Mode. */
4904	VMXDisable();
4905
4906	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4907
4908	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4909	CPUMSetHyperEIP(pVCpu, enmOp);
4910	for (int i = (int)cbParam - 1; i >= 0; i--)
4911	CPUMPushHyper(pVCpu, paParam[i]);
4912
4913	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4914
4915	/* Call the switcher. */
4916	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4917	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4918
4919	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4920	/* Make sure the VMX instructions don't cause #UD faults. */
4921	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4922
4923	/* Re-enter VMX Root Mode */
4924	rc2 = VMXEnable(HCPhysCpuPage);
4925	if (RT_FAILURE(rc2))
4926	{
4927	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4928	ASMSetFlags(uOldEflags);
4929	return rc2;
4930	}
4931
4932	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4933	AssertRC(rc2);
4934	Assert(!(ASMGetFlags() & X86_EFL_IF));
4935	ASMSetFlags(uOldEflags);
4936	return rc;
4937	}
4938
4939
4940	/**
4941	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4942	* supporting 64-bit guests.
4943	*
4944	* @returns VBox status code.
4945	* @param fResume Whether to VMLAUNCH or VMRESUME.
4946	* @param pCtx Pointer to the guest-CPU context.
4947	* @param pCache Pointer to the VMCS cache.
4948	* @param pVM Pointer to the VM.
4949	* @param pVCpu Pointer to the VMCPU.
4950	*/
4951	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4952	{
4953	uint32_t aParam[6];
4954	PHMGLOBALCPUINFO pCpu = NULL;
4955	RTHCPHYS HCPhysCpuPage = 0;
4956	int rc = VERR_INTERNAL_ERROR_5;
4957
4958	pCpu = HMR0GetCurrentCpu();
4959	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4960
4961	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4962	pCache->uPos = 1;
4963	pCache->interPD = PGMGetInterPaeCR3(pVM);
4964	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4965	#endif
4966
4967	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4968	pCache->TestIn.HCPhysCpuPage = 0;
4969	pCache->TestIn.HCPhysVmcs = 0;
4970	pCache->TestIn.pCache = 0;
4971	pCache->TestOut.HCPhysVmcs = 0;
4972	pCache->TestOut.pCache = 0;
4973	pCache->TestOut.pCtx = 0;
4974	pCache->TestOut.eflags = 0;
4975	#endif
4976
4977	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4978	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4979	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4980	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4981	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4982	aParam[5] = 0;
4983
4984	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4985	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4986	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4987	#endif
4988	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4989
4990	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4991	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4992	Assert(pCtx->dr[4] == 10);
4993	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4994	#endif
4995
4996	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4997	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4998	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4999	pVCpu->hm.s.vmx.HCPhysVmcs));
5000	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5001	pCache->TestOut.HCPhysVmcs));
5002	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5003	pCache->TestOut.pCache));
5004	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5005	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5006	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5007	pCache->TestOut.pCtx));
5008	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5009	#endif
5010	return rc;
5011	}
5012
5013
5014	/**
5015	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5016	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5017	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5018	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5019	*
5020	* @returns VBox status code.
5021	* @param pVM Pointer to the VM.
5022	* @param pVCpu Pointer to the VMCPU.
5023	*/
5024	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5025	{
5026	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5027	{ \
5028	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5029	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5030	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5031	++cReadFields; \
5032	}
5033
5034	AssertPtr(pVM);
5035	AssertPtr(pVCpu);
5036	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5037	uint32_t cReadFields = 0;
5038
5039	/*
5040	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5041	* and serve to indicate exceptions to the rules.
5042	*/
5043
5044	/* Guest-natural selector base fields. */
5045	#if 0
5046	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5047	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5048	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5049	#endif
5050	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5051	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5052	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5053	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5054	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5055	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5056	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5057	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5058	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5059	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5062	#if 0
5063	/* Unused natural width guest-state fields. */
5064	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5066	#endif
5067	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5069
5070	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5071	#if 0
5072	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5075	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5081	#endif
5082
5083	/* Natural width guest-state fields. */
5084	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5085	#if 0
5086	/* Currently unused field. */
5087	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5088	#endif
5089
5090	if (pVM->hm.s.fNestedPaging)
5091	{
5092	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5093	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5094	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5095	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5096	}
5097	else
5098	{
5099	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5100	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5101	}
5102
5103	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5104	return VINF_SUCCESS;
5105	}
5106
5107
5108	/**
5109	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5110	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5111	* darwin, running 64-bit guests).
5112	*
5113	* @returns VBox status code.
5114	* @param pVCpu Pointer to the VMCPU.
5115	* @param idxField The VMCS field encoding.
5116	* @param u64Val 16, 32 or 64-bit value.
5117	*/
5118	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5119	{
5120	int rc;
5121	switch (idxField)
5122	{
5123	/*
5124	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5125	*/
5126	/* 64-bit Control fields. */
5127	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5128	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5129	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5130	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5131	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5132	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5133	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5134	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5135	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5136	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5137	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5138	case VMX_VMCS64_CTRL_EPTP_FULL:
5139	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5140	/* 64-bit Guest-state fields. */
5141	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5142	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5143	case VMX_VMCS64_GUEST_PAT_FULL:
5144	case VMX_VMCS64_GUEST_EFER_FULL:
5145	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5146	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5147	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5148	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5149	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5150	/* 64-bit Host-state fields. */
5151	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5152	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5153	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5154	{
5155	rc = VMXWriteVmcs32(idxField, u64Val);
5156	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5157	break;
5158	}
5159
5160	/*
5161	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5162	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5163	*/
5164	/* Natural-width Guest-state fields. */
5165	case VMX_VMCS_GUEST_CR3:
5166	case VMX_VMCS_GUEST_ES_BASE:
5167	case VMX_VMCS_GUEST_CS_BASE:
5168	case VMX_VMCS_GUEST_SS_BASE:
5169	case VMX_VMCS_GUEST_DS_BASE:
5170	case VMX_VMCS_GUEST_FS_BASE:
5171	case VMX_VMCS_GUEST_GS_BASE:
5172	case VMX_VMCS_GUEST_LDTR_BASE:
5173	case VMX_VMCS_GUEST_TR_BASE:
5174	case VMX_VMCS_GUEST_GDTR_BASE:
5175	case VMX_VMCS_GUEST_IDTR_BASE:
5176	case VMX_VMCS_GUEST_RSP:
5177	case VMX_VMCS_GUEST_RIP:
5178	case VMX_VMCS_GUEST_SYSENTER_ESP:
5179	case VMX_VMCS_GUEST_SYSENTER_EIP:
5180	{
5181	if (!(u64Val >> 32))
5182	{
5183	/* If this field is 64-bit, VT-x will zero out the top bits. */
5184	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5185	}
5186	else
5187	{
5188	/* Assert that only the 32->64 switcher case should ever come here. */
5189	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5190	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5191	}
5192	break;
5193	}
5194
5195	default:
5196	{
5197	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5198	rc = VERR_INVALID_PARAMETER;
5199	break;
5200	}
5201	}
5202	AssertRCReturn(rc, rc);
5203	return rc;
5204	}
5205
5206
5207	/**
5208	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5209	* hosts (except darwin) for 64-bit guests.
5210	*
5211	* @param pVCpu Pointer to the VMCPU.
5212	* @param idxField The VMCS field encoding.
5213	* @param u64Val 16, 32 or 64-bit value.
5214	*/
5215	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5216	{
5217	AssertPtr(pVCpu);
5218	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5219
5220	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5221	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5222
5223	/* Make sure there are no duplicates. */
5224	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5225	{
5226	if (pCache->Write.aField[i] == idxField)
5227	{
5228	pCache->Write.aFieldVal[i] = u64Val;
5229	return VINF_SUCCESS;
5230	}
5231	}
5232
5233	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5234	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5235	pCache->Write.cValidEntries++;
5236	return VINF_SUCCESS;
5237	}
5238
5239	/* Enable later when the assembly code uses these as callbacks. */
5240	#if 0
5241	/*
5242	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5243	*
5244	* @param pVCpu Pointer to the VMCPU.
5245	* @param pCache Pointer to the VMCS cache.
5246	*
5247	* @remarks No-long-jump zone!!!
5248	*/
5249	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5250	{
5251	AssertPtr(pCache);
5252	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5253	{
5254	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5255	AssertRC(rc);
5256	}
5257	pCache->Write.cValidEntries = 0;
5258	}
5259
5260
5261	/**
5262	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5263	*
5264	* @param pVCpu Pointer to the VMCPU.
5265	* @param pCache Pointer to the VMCS cache.
5266	*
5267	* @remarks No-long-jump zone!!!
5268	*/
5269	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5270	{
5271	AssertPtr(pCache);
5272	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5273	{
5274	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5275	AssertRC(rc);
5276	}
5277	}
5278	#endif
5279	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5280
5281
5282	/**
5283	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5284	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5285	* timer.
5286	*
5287	* @returns VBox status code.
5288	* @param pVCpu Pointer to the VMCPU.
5289	*
5290	* @remarks No-long-jump zone!!!
5291	*/
5292	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5293	{
5294	int rc = VERR_INTERNAL_ERROR_5;
5295	bool fOffsettedTsc = false;
5296	PVM pVM = pVCpu->CTX_SUFF(pVM);
5297	if (pVM->hm.s.vmx.fUsePreemptTimer)
5298	{
5299	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5300
5301	/* Make sure the returned values have sane upper and lower boundaries. */
5302	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5303	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5304	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5305	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5306
5307	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5308	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5309	}
5310	else
5311	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5312
5313	if (fOffsettedTsc)
5314	{
5315	uint64_t u64CurTSC = ASMReadTSC();
5316	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5317	{
5318	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5319	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5320
5321	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5322	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5323	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5324	}
5325	else
5326	{
5327	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5328	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5329	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5331	}
5332	}
5333	else
5334	{
5335	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5336	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5337	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5338	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5339	}
5340	}
5341
5342
5343	/**
5344	* Determines if an exception is a contributory exception. Contributory
5345	* exceptions are ones which can cause double-faults. Page-fault is
5346	* intentionally not included here as it's a conditional contributory exception.
5347	*
5348	* @returns true if the exception is contributory, false otherwise.
5349	* @param uVector The exception vector.
5350	*/
5351	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5352	{
5353	switch (uVector)
5354	{
5355	case X86_XCPT_GP:
5356	case X86_XCPT_SS:
5357	case X86_XCPT_NP:
5358	case X86_XCPT_TS:
5359	case X86_XCPT_DE:
5360	return true;
5361	default:
5362	break;
5363	}
5364	return false;
5365	}
5366
5367
5368	/**
5369	* Sets an event as a pending event to be injected into the guest.
5370	*
5371	* @param pVCpu Pointer to the VMCPU.
5372	* @param u32IntInfo The VM-entry interruption-information field.
5373	* @param cbInstr The VM-entry instruction length in bytes (for software
5374	* interrupts, exceptions and privileged software
5375	* exceptions).
5376	* @param u32ErrCode The VM-entry exception error code.
5377	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5378	* page-fault.
5379	*
5380	* @remarks Statistics counter assumes this is a guest event being injected or
5381	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5382	* always incremented.
5383	*/
5384	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5385	RTGCUINTPTR GCPtrFaultAddress)
5386	{
5387	Assert(!pVCpu->hm.s.Event.fPending);
5388	pVCpu->hm.s.Event.fPending = true;
5389	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5390	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5391	pVCpu->hm.s.Event.cbInstr = cbInstr;
5392	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5393
5394	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5395	}
5396
5397
5398	/**
5399	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5400	*
5401	* @param pVCpu Pointer to the VMCPU.
5402	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5403	* out-of-sync. Make sure to update the required fields
5404	* before using them.
5405	*/
5406	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5407	{
5408	NOREF(pMixedCtx);
5409	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5410	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5411	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5412	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5413	}
5414
5415
5416	/**
5417	* Handle a condition that occurred while delivering an event through the guest
5418	* IDT.
5419	*
5420	* @returns VBox status code (informational error codes included).
5421	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5422	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5423	* continue execution of the guest which will delivery the #DF.
5424	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5425	*
5426	* @param pVCpu Pointer to the VMCPU.
5427	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5428	* out-of-sync. Make sure to update the required fields
5429	* before using them.
5430	* @param pVmxTransient Pointer to the VMX transient structure.
5431	*
5432	* @remarks No-long-jump zone!!!
5433	*/
5434	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5435	{
5436	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5437	AssertRCReturn(rc, rc);
5438	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5439	{
5440	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5441	AssertRCReturn(rc, rc);
5442
5443	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5444	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5445	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5446
5447	typedef enum
5448	{
5449	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5450	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5451	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5452	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5453	} VMXREFLECTXCPT;
5454
5455	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5456	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5457	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5458	{
5459	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5460	{
5461	enmReflect = VMXREFLECTXCPT_XCPT;
5462	#ifdef VBOX_STRICT
5463	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5464	&& uExitVector == X86_XCPT_PF)
5465	{
5466	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5467	}
5468	#endif
5469	if ( uExitVector == X86_XCPT_PF
5470	&& uIdtVector == X86_XCPT_PF)
5471	{
5472	pVmxTransient->fVectoringPF = true;
5473	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5474	}
5475	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5476	&& hmR0VmxIsContributoryXcpt(uExitVector)
5477	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5478	\|\| uIdtVector == X86_XCPT_PF))
5479	{
5480	enmReflect = VMXREFLECTXCPT_DF;
5481	}
5482	else if (uIdtVector == X86_XCPT_DF)
5483	enmReflect = VMXREFLECTXCPT_TF;
5484	}
5485	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5486	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5487	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5488	{
5489	/*
5490	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5491	* (whatever they are) as they reoccur when restarting the instruction.
5492	*/
5493	enmReflect = VMXREFLECTXCPT_XCPT;
5494	}
5495	}
5496	else
5497	{
5498	/*
5499	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5500	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5501	* original exception to the guest after handling the VM-exit.
5502	*/
5503	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5504	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5505	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5506	{
5507	enmReflect = VMXREFLECTXCPT_XCPT;
5508	}
5509	}
5510
5511	switch (enmReflect)
5512	{
5513	case VMXREFLECTXCPT_XCPT:
5514	{
5515	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5516	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5517	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5518
5519	uint32_t u32ErrCode = 0;
5520	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5521	{
5522	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5523	AssertRCReturn(rc, rc);
5524	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5525	}
5526
5527	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5528	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5529	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5530	rc = VINF_SUCCESS;
5531	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5532	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5533
5534	break;
5535	}
5536
5537	case VMXREFLECTXCPT_DF:
5538	{
5539	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5540	rc = VINF_HM_DOUBLE_FAULT;
5541	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5542	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5543
5544	break;
5545	}
5546
5547	case VMXREFLECTXCPT_TF:
5548	{
5549	rc = VINF_EM_RESET;
5550	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5551	uExitVector));
5552	break;
5553	}
5554
5555	default:
5556	Assert(rc == VINF_SUCCESS);
5557	break;
5558	}
5559	}
5560	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5561	return rc;
5562	}
5563
5564
5565	/**
5566	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5567	*
5568	* @returns VBox status code.
5569	* @param pVCpu Pointer to the VMCPU.
5570	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5571	* out-of-sync. Make sure to update the required fields
5572	* before using them.
5573	*
5574	* @remarks No-long-jump zone!!!
5575	*/
5576	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5577	{
5578	NOREF(pMixedCtx);
5579
5580	/* Since this can be called from our preemption hook it's safer to make the guest-CR0 update non-preemptible. */
5581	VMMRZCallRing3Disable(pVCpu);
5582	HM_DISABLE_PREEMPT_IF_NEEDED();
5583
5584	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5585	{
5586	uint32_t uVal = 0;
5587	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5588	AssertRCReturn(rc, rc);
5589
5590	uint32_t uShadow = 0;
5591	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5592	AssertRCReturn(rc, rc);
5593
5594	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5595	CPUMSetGuestCR0(pVCpu, uVal);
5596	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5597	}
5598
5599	HM_RESTORE_PREEMPT_IF_NEEDED();
5600	VMMRZCallRing3Enable(pVCpu);
5601
5602	return VINF_SUCCESS;
5603	}
5604
5605
5606	/**
5607	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5608	*
5609	* @returns VBox status code.
5610	* @param pVCpu Pointer to the VMCPU.
5611	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5612	* out-of-sync. Make sure to update the required fields
5613	* before using them.
5614	*
5615	* @remarks No-long-jump zone!!!
5616	*/
5617	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5618	{
5619	NOREF(pMixedCtx);
5620
5621	int rc = VINF_SUCCESS;
5622	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5623	{
5624	uint32_t uVal = 0;
5625	uint32_t uShadow = 0;
5626	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5627	AssertRCReturn(rc, rc);
5628	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5629	AssertRCReturn(rc, rc);
5630
5631	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5632	CPUMSetGuestCR4(pVCpu, uVal);
5633	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5634	}
5635	return rc;
5636	}
5637
5638
5639	/**
5640	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5641	*
5642	* @returns VBox status code.
5643	* @param pVCpu Pointer to the VMCPU.
5644	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5645	* out-of-sync. Make sure to update the required fields
5646	* before using them.
5647	*
5648	* @remarks No-long-jump zone!!!
5649	*/
5650	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5651	{
5652	int rc = VINF_SUCCESS;
5653	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5654	{
5655	uint64_t u64Val = 0;
5656	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5657	AssertRCReturn(rc, rc);
5658
5659	pMixedCtx->rip = u64Val;
5660	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5661	}
5662	return rc;
5663	}
5664
5665
5666	/**
5667	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5668	*
5669	* @returns VBox status code.
5670	* @param pVCpu Pointer to the VMCPU.
5671	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5672	* out-of-sync. Make sure to update the required fields
5673	* before using them.
5674	*
5675	* @remarks No-long-jump zone!!!
5676	*/
5677	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5678	{
5679	int rc = VINF_SUCCESS;
5680	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5681	{
5682	uint64_t u64Val = 0;
5683	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5684	AssertRCReturn(rc, rc);
5685
5686	pMixedCtx->rsp = u64Val;
5687	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5688	}
5689	return rc;
5690	}
5691
5692
5693	/**
5694	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5695	*
5696	* @returns VBox status code.
5697	* @param pVCpu Pointer to the VMCPU.
5698	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5699	* out-of-sync. Make sure to update the required fields
5700	* before using them.
5701	*
5702	* @remarks No-long-jump zone!!!
5703	*/
5704	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5705	{
5706	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5707	{
5708	uint32_t uVal = 0;
5709	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5710	AssertRCReturn(rc, rc);
5711
5712	pMixedCtx->eflags.u32 = uVal;
5713	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5714	{
5715	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5716	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5717
5718	pMixedCtx->eflags.Bits.u1VM = 0;
5719	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5720	}
5721
5722	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5723	}
5724	return VINF_SUCCESS;
5725	}
5726
5727
5728	/**
5729	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5730	* guest-CPU context.
5731	*/
5732	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5733	{
5734	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5735	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5736	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5737	return rc;
5738	}
5739
5740
5741	/**
5742	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5743	* from the guest-state area in the VMCS.
5744	*
5745	* @param pVCpu Pointer to the VMCPU.
5746	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5747	* out-of-sync. Make sure to update the required fields
5748	* before using them.
5749	*
5750	* @remarks No-long-jump zone!!!
5751	*/
5752	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5753	{
5754	uint32_t uIntrState = 0;
5755	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5756	AssertRC(rc);
5757
5758	if (!uIntrState)
5759	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5760	else
5761	{
5762	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5763	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5764	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5765	AssertRC(rc);
5766	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5767	AssertRC(rc);
5768
5769	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5770	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5771	}
5772	}
5773
5774
5775	/**
5776	* Saves the guest's activity state.
5777	*
5778	* @returns VBox status code.
5779	* @param pVCpu Pointer to the VMCPU.
5780	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5781	* out-of-sync. Make sure to update the required fields
5782	* before using them.
5783	*
5784	* @remarks No-long-jump zone!!!
5785	*/
5786	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5787	{
5788	NOREF(pMixedCtx);
5789	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5790	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5791	return VINF_SUCCESS;
5792	}
5793
5794
5795	/**
5796	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5797	* the current VMCS into the guest-CPU context.
5798	*
5799	* @returns VBox status code.
5800	* @param pVCpu Pointer to the VMCPU.
5801	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5802	* out-of-sync. Make sure to update the required fields
5803	* before using them.
5804	*
5805	* @remarks No-long-jump zone!!!
5806	*/
5807	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5808	{
5809	int rc = VINF_SUCCESS;
5810	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5811	{
5812	uint32_t u32Val = 0;
5813	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5814	pMixedCtx->SysEnter.cs = u32Val;
5815	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5816	}
5817
5818	uint64_t u64Val = 0;
5819	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5820	{
5821	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5822	pMixedCtx->SysEnter.eip = u64Val;
5823	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5824	}
5825	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5826	{
5827	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5828	pMixedCtx->SysEnter.esp = u64Val;
5829	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5830	}
5831	return rc;
5832	}
5833
5834
5835	/**
5836	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5837	* the CPU back into the guest-CPU context.
5838	*
5839	* @returns VBox status code.
5840	* @param pVCpu Pointer to the VMCPU.
5841	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5842	* out-of-sync. Make sure to update the required fields
5843	* before using them.
5844	*
5845	* @remarks No-long-jump zone!!!
5846	*/
5847	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5848	{
5849	#if HC_ARCH_BITS == 64
5850	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5851	{
5852	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5853	VMMRZCallRing3Disable(pVCpu);
5854	HM_DISABLE_PREEMPT_IF_NEEDED();
5855
5856	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5857	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5858	{
5859	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5860	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5861	}
5862
5863	HM_RESTORE_PREEMPT_IF_NEEDED();
5864	VMMRZCallRing3Enable(pVCpu);
5865	}
5866	else
5867	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5868	#else
5869	NOREF(pMixedCtx);
5870	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5871	#endif
5872
5873	return VINF_SUCCESS;
5874	}
5875
5876
5877	/**
5878	* Saves the auto load/store'd guest MSRs from the current VMCS into
5879	* the guest-CPU context.
5880	*
5881	* @returns VBox status code.
5882	* @param pVCpu Pointer to the VMCPU.
5883	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5884	* out-of-sync. Make sure to update the required fields
5885	* before using them.
5886	*
5887	* @remarks No-long-jump zone!!!
5888	*/
5889	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5890	{
5891	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5892	return VINF_SUCCESS;
5893
5894	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5895	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5896	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5897	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5898	{
5899	switch (pMsr->u32Msr)
5900	{
5901	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
5902	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5903	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5904	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5905	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5906	default:
5907	{
5908	AssertFailed();
5909	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5910	}
5911	}
5912	}
5913
5914	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5915	return VINF_SUCCESS;
5916	}
5917
5918
5919	/**
5920	* Saves the guest control registers from the current VMCS into the guest-CPU
5921	* context.
5922	*
5923	* @returns VBox status code.
5924	* @param pVCpu Pointer to the VMCPU.
5925	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5926	* out-of-sync. Make sure to update the required fields
5927	* before using them.
5928	*
5929	* @remarks No-long-jump zone!!!
5930	*/
5931	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5932	{
5933	/* Guest CR0. Guest FPU. */
5934	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5935	AssertRCReturn(rc, rc);
5936
5937	/* Guest CR4. */
5938	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5939	AssertRCReturn(rc, rc);
5940
5941	/* Guest CR2 - updated always during the world-switch or in #PF. */
5942	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5943	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5944	{
5945	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5946	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5947
5948	PVM pVM = pVCpu->CTX_SUFF(pVM);
5949	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5950	\|\| ( pVM->hm.s.fNestedPaging
5951	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5952	{
5953	uint64_t u64Val = 0;
5954	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5955	if (pMixedCtx->cr3 != u64Val)
5956	{
5957	CPUMSetGuestCR3(pVCpu, u64Val);
5958	if (VMMRZCallRing3IsEnabled(pVCpu))
5959	{
5960	PGMUpdateCR3(pVCpu, u64Val);
5961	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5962	}
5963	else
5964	{
5965	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5966	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5967	}
5968	}
5969
5970	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5971	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5972	{
5973	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5974	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5975	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5976	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5977
5978	if (VMMRZCallRing3IsEnabled(pVCpu))
5979	{
5980	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5981	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5982	}
5983	else
5984	{
5985	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5986	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5987	}
5988	}
5989	}
5990
5991	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
5992	}
5993
5994	/*
5995	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5996	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5997	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5998	*
5999	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6000	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6001	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6002	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6003	*
6004	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6005	*/
6006	if (VMMRZCallRing3IsEnabled(pVCpu))
6007	{
6008	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6009	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6010
6011	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6012	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6013
6014	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6015	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6016	}
6017
6018	return rc;
6019	}
6020
6021
6022	/**
6023	* Reads a guest segment register from the current VMCS into the guest-CPU
6024	* context.
6025	*
6026	* @returns VBox status code.
6027	* @param pVCpu Pointer to the VMCPU.
6028	* @param idxSel Index of the selector in the VMCS.
6029	* @param idxLimit Index of the segment limit in the VMCS.
6030	* @param idxBase Index of the segment base in the VMCS.
6031	* @param idxAccess Index of the access rights of the segment in the VMCS.
6032	* @param pSelReg Pointer to the segment selector.
6033	*
6034	* @remarks No-long-jump zone!!!
6035	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6036	* macro as that takes care of whether to read from the VMCS cache or
6037	* not.
6038	*/
6039	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6040	PCPUMSELREG pSelReg)
6041	{
6042	NOREF(pVCpu);
6043
6044	uint32_t u32Val = 0;
6045	int rc = VMXReadVmcs32(idxSel, &u32Val);
6046	AssertRCReturn(rc, rc);
6047	pSelReg->Sel = (uint16_t)u32Val;
6048	pSelReg->ValidSel = (uint16_t)u32Val;
6049	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6050
6051	rc = VMXReadVmcs32(idxLimit, &u32Val);
6052	AssertRCReturn(rc, rc);
6053	pSelReg->u32Limit = u32Val;
6054
6055	uint64_t u64Val = 0;
6056	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6057	AssertRCReturn(rc, rc);
6058	pSelReg->u64Base = u64Val;
6059
6060	rc = VMXReadVmcs32(idxAccess, &u32Val);
6061	AssertRCReturn(rc, rc);
6062	pSelReg->Attr.u = u32Val;
6063
6064	/*
6065	* If VT-x marks the segment as unusable, most other bits remain undefined:
6066	* - For CS the L, D and G bits have meaning.
6067	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6068	* - For the remaining data segments no bits are defined.
6069	*
6070	* The present bit and the unusable bit has been observed to be set at the
6071	* same time (the selector was supposed to invalid as we started executing
6072	* a V8086 interrupt in ring-0).
6073	*
6074	* What should be important for the rest of the VBox code, is that the P bit is
6075	* cleared. Some of the other VBox code recognizes the unusable bit, but
6076	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6077	* safe side here, we'll strip off P and other bits we don't care about. If
6078	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6079	*
6080	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6081	*/
6082	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6083	{
6084	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6085
6086	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6087	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6088	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6089
6090	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6091	#ifdef DEBUG_bird
6092	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6093	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6094	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6095	#endif
6096	}
6097	return VINF_SUCCESS;
6098	}
6099
6100
6101	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6102	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6103	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6104	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6105	#else
6106	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6107	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6108	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6109	#endif
6110
6111
6112	/**
6113	* Saves the guest segment registers from the current VMCS into the guest-CPU
6114	* context.
6115	*
6116	* @returns VBox status code.
6117	* @param pVCpu Pointer to the VMCPU.
6118	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6119	* out-of-sync. Make sure to update the required fields
6120	* before using them.
6121	*
6122	* @remarks No-long-jump zone!!!
6123	*/
6124	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6125	{
6126	/* Guest segment registers. */
6127	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6128	{
6129	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6130	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6131	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6132	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6133	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6134	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6135	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6136
6137	/* Restore segment attributes for real-on-v86 mode hack. */
6138	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6139	{
6140	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6141	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6142	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6143	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6144	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6145	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6146	}
6147	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6148	}
6149
6150	return VINF_SUCCESS;
6151	}
6152
6153
6154	/**
6155	* Saves the guest descriptor table registers and task register from the current
6156	* VMCS into the guest-CPU context.
6157	*
6158	* @returns VBox status code.
6159	* @param pVCpu Pointer to the VMCPU.
6160	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6161	* out-of-sync. Make sure to update the required fields
6162	* before using them.
6163	*
6164	* @remarks No-long-jump zone!!!
6165	*/
6166	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6167	{
6168	int rc = VINF_SUCCESS;
6169
6170	/* Guest LDTR. */
6171	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6172	{
6173	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6174	AssertRCReturn(rc, rc);
6175	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6176	}
6177
6178	/* Guest GDTR. */
6179	uint64_t u64Val = 0;
6180	uint32_t u32Val = 0;
6181	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6182	{
6183	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6184	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6185	pMixedCtx->gdtr.pGdt = u64Val;
6186	pMixedCtx->gdtr.cbGdt = u32Val;
6187	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6188	}
6189
6190	/* Guest IDTR. */
6191	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6192	{
6193	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6194	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6195	pMixedCtx->idtr.pIdt = u64Val;
6196	pMixedCtx->idtr.cbIdt = u32Val;
6197	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6198	}
6199
6200	/* Guest TR. */
6201	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6202	{
6203	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6204	AssertRCReturn(rc, rc);
6205
6206	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6207	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6208	{
6209	rc = VMXLOCAL_READ_SEG(TR, tr);
6210	AssertRCReturn(rc, rc);
6211	}
6212	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6213	}
6214	return rc;
6215	}
6216
6217	#undef VMXLOCAL_READ_SEG
6218
6219
6220	/**
6221	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6222	* context.
6223	*
6224	* @returns VBox status code.
6225	* @param pVCpu Pointer to the VMCPU.
6226	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6227	* out-of-sync. Make sure to update the required fields
6228	* before using them.
6229	*
6230	* @remarks No-long-jump zone!!!
6231	*/
6232	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6233	{
6234	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6235	{
6236	if (!pVCpu->hm.s.fUsingHyperDR7)
6237	{
6238	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6239	uint32_t u32Val;
6240	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6241	pMixedCtx->dr[7] = u32Val;
6242	}
6243
6244	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6245	}
6246	return VINF_SUCCESS;
6247	}
6248
6249
6250	/**
6251	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6252	*
6253	* @returns VBox status code.
6254	* @param pVCpu Pointer to the VMCPU.
6255	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6256	* out-of-sync. Make sure to update the required fields
6257	* before using them.
6258	*
6259	* @remarks No-long-jump zone!!!
6260	*/
6261	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6262	{
6263	NOREF(pMixedCtx);
6264
6265	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6266	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6267	return VINF_SUCCESS;
6268	}
6269
6270
6271	/**
6272	* Saves the entire guest state from the currently active VMCS into the
6273	* guest-CPU context. This essentially VMREADs all guest-data.
6274	*
6275	* @returns VBox status code.
6276	* @param pVCpu Pointer to the VMCPU.
6277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6278	* out-of-sync. Make sure to update the required fields
6279	* before using them.
6280	*/
6281	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282	{
6283	Assert(pVCpu);
6284	Assert(pMixedCtx);
6285
6286	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6287	return VINF_SUCCESS;
6288
6289	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6290	again on the ring-3 callback path, there is no real need to. */
6291	if (VMMRZCallRing3IsEnabled(pVCpu))
6292	VMMR0LogFlushDisable(pVCpu);
6293	else
6294	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6295	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6296
6297	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6298	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6299
6300	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6301	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6302
6303	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6304	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6305
6306	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6308
6309	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6311
6312	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6320
6321	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6323
6324	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6326
6327	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6328	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6329
6330	if (VMMRZCallRing3IsEnabled(pVCpu))
6331	VMMR0LogFlushEnable(pVCpu);
6332
6333	return rc;
6334	}
6335
6336
6337	/**
6338	* Check per-VM and per-VCPU force flag actions that require us to go back to
6339	* ring-3 for one reason or another.
6340	*
6341	* @returns VBox status code (information status code included).
6342	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6343	* ring-3.
6344	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6345	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6346	* interrupts)
6347	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6348	* all EMTs to be in ring-3.
6349	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6350	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6351	* to the EM loop.
6352	*
6353	* @param pVM Pointer to the VM.
6354	* @param pVCpu Pointer to the VMCPU.
6355	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6356	* out-of-sync. Make sure to update the required fields
6357	* before using them.
6358	*/
6359	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6360	{
6361	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6362
6363	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6364	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6365	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6366	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6367	{
6368	/* We need the control registers now, make sure the guest-CPU context is updated. */
6369	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6370	AssertRCReturn(rc3, rc3);
6371
6372	/* Pending HM CR3 sync. */
6373	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6374	{
6375	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6376	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6377	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6378	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6379	}
6380
6381	/* Pending HM PAE PDPEs. */
6382	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6383	{
6384	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6385	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6386	}
6387
6388	/* Pending PGM C3 sync. */
6389	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6390	{
6391	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6392	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6393	if (rc2 != VINF_SUCCESS)
6394	{
6395	AssertRC(rc2);
6396	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6397	return rc2;
6398	}
6399	}
6400
6401	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6402	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6403	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6404	{
6405	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6406	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6407	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6408	return rc2;
6409	}
6410
6411	/* Pending VM request packets, such as hardware interrupts. */
6412	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6413	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6414	{
6415	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6416	return VINF_EM_PENDING_REQUEST;
6417	}
6418
6419	/* Pending PGM pool flushes. */
6420	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6421	{
6422	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6423	return VINF_PGM_POOL_FLUSH_PENDING;
6424	}
6425
6426	/* Pending DMA requests. */
6427	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6428	{
6429	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6430	return VINF_EM_RAW_TO_R3;
6431	}
6432	}
6433
6434	return VINF_SUCCESS;
6435	}
6436
6437
6438	/**
6439	* Converts any TRPM trap into a pending HM event. This is typically used when
6440	* entering from ring-3 (not longjmp returns).
6441	*
6442	* @param pVCpu Pointer to the VMCPU.
6443	*/
6444	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6445	{
6446	Assert(TRPMHasTrap(pVCpu));
6447	Assert(!pVCpu->hm.s.Event.fPending);
6448
6449	uint8_t uVector;
6450	TRPMEVENT enmTrpmEvent;
6451	RTGCUINT uErrCode;
6452	RTGCUINTPTR GCPtrFaultAddress;
6453	uint8_t cbInstr;
6454
6455	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6456	AssertRC(rc);
6457
6458	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6459	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6460	if (enmTrpmEvent == TRPM_TRAP)
6461	{
6462	switch (uVector)
6463	{
6464	case X86_XCPT_BP:
6465	case X86_XCPT_OF:
6466	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6467	break;
6468
6469	case X86_XCPT_PF:
6470	case X86_XCPT_DF:
6471	case X86_XCPT_TS:
6472	case X86_XCPT_NP:
6473	case X86_XCPT_SS:
6474	case X86_XCPT_GP:
6475	case X86_XCPT_AC:
6476	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6477	/* no break! */
6478	default:
6479	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6480	break;
6481	}
6482	}
6483	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6484	{
6485	if (uVector == X86_XCPT_NMI)
6486	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6487	else
6488	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6489	}
6490	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6491	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6492	else
6493	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6494
6495	rc = TRPMResetTrap(pVCpu);
6496	AssertRC(rc);
6497	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6498	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6499
6500	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6501	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6502	}
6503
6504
6505	/**
6506	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6507	* VT-x to execute any instruction.
6508	*
6509	* @param pvCpu Pointer to the VMCPU.
6510	*/
6511	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6512	{
6513	Assert(pVCpu->hm.s.Event.fPending);
6514
6515	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6516	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6517	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6518	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6519
6520	/* If a trap was already pending, we did something wrong! */
6521	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6522
6523	TRPMEVENT enmTrapType;
6524	switch (uVectorType)
6525	{
6526	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6527	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6528	enmTrapType = TRPM_HARDWARE_INT;
6529	break;
6530
6531	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6532	enmTrapType = TRPM_SOFTWARE_INT;
6533	break;
6534
6535	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6536	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6537	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6538	enmTrapType = TRPM_TRAP;
6539	break;
6540
6541	default:
6542	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6543	enmTrapType = TRPM_32BIT_HACK;
6544	break;
6545	}
6546
6547	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6548
6549	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6550	AssertRC(rc);
6551
6552	if (fErrorCodeValid)
6553	TRPMSetErrorCode(pVCpu, uErrorCode);
6554
6555	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6556	&& uVector == X86_XCPT_PF)
6557	{
6558	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6559	}
6560	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6561	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6562	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6563	{
6564	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6565	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6566	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6567	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6568	}
6569	pVCpu->hm.s.Event.fPending = false;
6570	}
6571
6572
6573	/**
6574	* Does the necessary state syncing before returning to ring-3 for any reason
6575	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6576	*
6577	* @returns VBox status code.
6578	* @param pVM Pointer to the VM.
6579	* @param pVCpu Pointer to the VMCPU.
6580	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6581	* be out-of-sync. Make sure to update the required
6582	* fields before using them.
6583	* @param fSaveGuestState Whether to save the guest state or not.
6584	*
6585	* @remarks No-long-jmp zone!!!
6586	*/
6587	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6588	{
6589	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6590	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6591
6592	RTCPUID idCpu = RTMpCpuId();
6593	Log4Func(("HostCpuId=%u\n", idCpu));
6594
6595	/*
6596	* !!! IMPORTANT !!!
6597	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6598	*/
6599
6600	/* Save the guest state if necessary. */
6601	if ( fSaveGuestState
6602	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6603	{
6604	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6605	AssertRCReturn(rc, rc);
6606	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6607	}
6608
6609	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6610	if (CPUMIsGuestFPUStateActive(pVCpu))
6611	{
6612	/* We shouldn't reload CR0 without saving it first. */
6613	if (!fSaveGuestState)
6614	{
6615	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6616	AssertRCReturn(rc, rc);
6617	}
6618	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6619	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6620	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6621	}
6622
6623	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6624	#ifdef VBOX_STRICT
6625	if (CPUMIsHyperDebugStateActive(pVCpu))
6626	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6627	#endif
6628	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6629	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6630	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6631	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6632
6633	#if HC_ARCH_BITS == 64
6634	/* Restore host-state bits that VT-x only restores partially. */
6635	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6636	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6637	{
6638	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6639	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6640	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6641	}
6642	#endif
6643
6644	#if HC_ARCH_BITS == 64
6645	/* Restore the host MSRs as we're leaving VT-x context. */
6646	if ( pVM->hm.s.fAllow64BitGuests
6647	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6648	{
6649	/* We shouldn't reload the guest MSRs without saving it first. */
6650	if (!fSaveGuestState)
6651	{
6652	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6653	AssertRCReturn(rc, rc);
6654	}
6655	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6656	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6657	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6658	}
6659	#endif
6660
6661	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6662	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6663	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6664	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6665	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6666	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6667	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6668	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6669
6670	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6671
6672	/** @todo This partially defeats the purpose of having preemption hooks.
6673	* The problem is, deregistering the hooks should be moved to a place that
6674	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6675	* context.
6676	*/
6677	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6678	{
6679	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6680	AssertRCReturn(rc, rc);
6681
6682	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6683	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6684	}
6685	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6686	NOREF(idCpu);
6687
6688	return VINF_SUCCESS;
6689	}
6690
6691
6692	/**
6693	* Leaves the VT-x session.
6694	*
6695	* @returns VBox status code.
6696	* @param pVM Pointer to the VM.
6697	* @param pVCpu Pointer to the VMCPU.
6698	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6699	* out-of-sync. Make sure to update the required fields
6700	* before using them.
6701	*
6702	* @remarks No-long-jmp zone!!!
6703	*/
6704	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6705	{
6706	HM_DISABLE_PREEMPT_IF_NEEDED();
6707	HMVMX_ASSERT_CPU_SAFE();
6708	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6709	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6710
6711	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6712	and done this from the VMXR0ThreadCtxCallback(). */
6713	if (!pVCpu->hm.s.fLeaveDone)
6714	{
6715	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6716	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6717	pVCpu->hm.s.fLeaveDone = true;
6718	}
6719
6720	/*
6721	* !!! IMPORTANT !!!
6722	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6723	*/
6724
6725	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6726	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6727	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6728	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6729	VMMR0ThreadCtxHooksDeregister(pVCpu);
6730
6731	/* Leave HM context. This takes care of local init (term). */
6732	int rc = HMR0LeaveCpu(pVCpu);
6733
6734	HM_RESTORE_PREEMPT_IF_NEEDED();
6735
6736	return rc;
6737	}
6738
6739
6740	/**
6741	* Does the necessary state syncing before doing a longjmp to ring-3.
6742	*
6743	* @returns VBox status code.
6744	* @param pVM Pointer to the VM.
6745	* @param pVCpu Pointer to the VMCPU.
6746	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6747	* out-of-sync. Make sure to update the required fields
6748	* before using them.
6749	*
6750	* @remarks No-long-jmp zone!!!
6751	*/
6752	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6753	{
6754	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6755	}
6756
6757
6758	/**
6759	* Take necessary actions before going back to ring-3.
6760	*
6761	* An action requires us to go back to ring-3. This function does the necessary
6762	* steps before we can safely return to ring-3. This is not the same as longjmps
6763	* to ring-3, this is voluntary and prepares the guest so it may continue
6764	* executing outside HM (recompiler/IEM).
6765	*
6766	* @returns VBox status code.
6767	* @param pVM Pointer to the VM.
6768	* @param pVCpu Pointer to the VMCPU.
6769	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6770	* out-of-sync. Make sure to update the required fields
6771	* before using them.
6772	* @param rcExit The reason for exiting to ring-3. Can be
6773	* VINF_VMM_UNKNOWN_RING3_CALL.
6774	*/
6775	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6776	{
6777	Assert(pVM);
6778	Assert(pVCpu);
6779	Assert(pMixedCtx);
6780	HMVMX_ASSERT_PREEMPT_SAFE();
6781
6782	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6783	{
6784	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6785	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6786	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6787	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6788	}
6789
6790	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6791	VMMRZCallRing3Disable(pVCpu);
6792	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6793
6794	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6795	if (pVCpu->hm.s.Event.fPending)
6796	{
6797	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6798	Assert(!pVCpu->hm.s.Event.fPending);
6799	}
6800
6801	/* Save guest state and restore host state bits. */
6802	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6803	AssertRCReturn(rc, rc);
6804	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6805
6806	/* Sync recompiler state. */
6807	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6808	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6809	\| CPUM_CHANGED_LDTR
6810	\| CPUM_CHANGED_GDTR
6811	\| CPUM_CHANGED_IDTR
6812	\| CPUM_CHANGED_TR
6813	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6814	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6815	if ( pVM->hm.s.fNestedPaging
6816	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6817	{
6818	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6819	}
6820
6821	Assert(!pVCpu->hm.s.fClearTrapFlag);
6822
6823	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6824	if (rcExit != VINF_EM_RAW_INTERRUPT)
6825	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6826
6827	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6828
6829	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6830	VMMRZCallRing3RemoveNotification(pVCpu);
6831	VMMRZCallRing3Enable(pVCpu);
6832
6833	return rc;
6834	}
6835
6836
6837	/**
6838	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6839	* longjump to ring-3 and possibly get preempted.
6840	*
6841	* @returns VBox status code.
6842	* @param pVCpu Pointer to the VMCPU.
6843	* @param enmOperation The operation causing the ring-3 longjump.
6844	* @param pvUser Opaque pointer to the guest-CPU context. The data
6845	* may be out-of-sync. Make sure to update the required
6846	* fields before using them.
6847	*/
6848	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6849	{
6850	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6851	{
6852	/*
6853	* !!! IMPORTANT !!!
6854	* If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6855	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6856	*/
6857	VMMRZCallRing3RemoveNotification(pVCpu);
6858	VMMRZCallRing3Disable(pVCpu);
6859	HM_DISABLE_PREEMPT_IF_NEEDED();
6860
6861	PVM pVM = pVCpu->CTX_SUFF(pVM);
6862	if (CPUMIsGuestFPUStateActive(pVCpu))
6863	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6864
6865	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6866
6867	#if HC_ARCH_BITS == 64
6868	/* Restore host-state bits that VT-x only restores partially. */
6869	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6870	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6871	{
6872	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6873	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6874	}
6875	/* Restore the host MSRs as we're leaving VT-x context. */
6876	if ( pVM->hm.s.fAllow64BitGuests
6877	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6878	{
6879	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6880	}
6881	#endif
6882	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6883	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6884	{
6885	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6886	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6887	}
6888
6889	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6890	VMMR0ThreadCtxHooksDeregister(pVCpu);
6891
6892	HMR0LeaveCpu(pVCpu);
6893	HM_RESTORE_PREEMPT_IF_NEEDED();
6894	return VINF_SUCCESS;
6895	}
6896
6897	Assert(pVCpu);
6898	Assert(pvUser);
6899	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6900	HMVMX_ASSERT_PREEMPT_SAFE();
6901
6902	VMMRZCallRing3Disable(pVCpu);
6903	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6904
6905	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6906	enmOperation));
6907
6908	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6909	AssertRCReturn(rc, rc);
6910
6911	VMMRZCallRing3Enable(pVCpu);
6912	return VINF_SUCCESS;
6913	}
6914
6915
6916	/**
6917	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6918	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6919	*
6920	* @param pVCpu Pointer to the VMCPU.
6921	*/
6922	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6923	{
6924	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6925	{
6926	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6927	{
6928	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6929	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6930	AssertRC(rc);
6931	}
6932	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6933	}
6934
6935
6936	/**
6937	* Evaluates the event to be delivered to the guest and sets it as the pending
6938	* event.
6939	*
6940	* @param pVCpu Pointer to the VMCPU.
6941	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6942	* out-of-sync. Make sure to update the required fields
6943	* before using them.
6944	*/
6945	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6946	{
6947	Assert(!pVCpu->hm.s.Event.fPending);
6948
6949	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6950	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6951	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6952	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6953
6954	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
6955	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6956	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6957	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6958	Assert(!TRPMHasTrap(pVCpu));
6959
6960	/** @todo SMI. SMIs take priority over NMIs. */
6961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6962	{
6963	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6964	if ( !fBlockMovSS
6965	&& !fBlockSti)
6966	{
6967	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6968	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6969	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6970	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6971
6972	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6973	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6974	}
6975	else
6976	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6977	}
6978	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6979	&& !pVCpu->hm.s.fSingleInstruction)
6980	{
6981	/*
6982	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6983	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6984	* evaluated here and not set as pending, solely based on the force-flags.
6985	*/
6986	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6987	AssertRC(rc);
6988	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6989	if ( !fBlockInt
6990	&& !fBlockSti
6991	&& !fBlockMovSS)
6992	{
6993	uint8_t u8Interrupt;
6994	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6995	if (RT_SUCCESS(rc))
6996	{
6997	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6998	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6999	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7000
7001	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7002	}
7003	else
7004	{
7005	/** @todo Does this actually happen? If not turn it into an assertion. */
7006	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7007	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7008	}
7009	}
7010	else
7011	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7012	}
7013	}
7014
7015
7016	/**
7017	* Sets a pending-debug exception to be delivered to the guest if the guest is
7018	* single-stepping.
7019	*
7020	* @param pVCpu Pointer to the VMCPU.
7021	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7022	* out-of-sync. Make sure to update the required fields
7023	* before using them.
7024	*/
7025	DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7026	{
7027	HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
7028	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7029	{
7030	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7031	AssertRC(rc);
7032	}
7033	}
7034
7035
7036	/**
7037	* Injects any pending events into the guest if the guest is in a state to
7038	* receive them.
7039	*
7040	* @returns VBox status code (informational status codes included).
7041	* @param pVCpu Pointer to the VMCPU.
7042	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7043	* out-of-sync. Make sure to update the required fields
7044	* before using them.
7045	*/
7046	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7047	{
7048	HMVMX_ASSERT_PREEMPT_SAFE();
7049	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7050
7051	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7052	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7053	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7054	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7055
7056	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7057	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7058	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7059	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7060	Assert(!TRPMHasTrap(pVCpu));
7061
7062	int rc = VINF_SUCCESS;
7063	if (pVCpu->hm.s.Event.fPending)
7064	{
7065	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7066	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7067	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7068	{
7069	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7070	AssertRCReturn(rc, rc);
7071	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7072	Assert(!fBlockInt);
7073	Assert(!fBlockSti);
7074	Assert(!fBlockMovSS);
7075	}
7076	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7077	{
7078	Assert(!fBlockSti);
7079	Assert(!fBlockMovSS);
7080	}
7081	#endif
7082	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7083	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7084	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7085	AssertRCReturn(rc, rc);
7086
7087	/* Update the interruptibility-state as it could have been changed by
7088	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7089	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7090	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7091
7092	#ifdef VBOX_WITH_STATISTICS
7093	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7094	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7095	else
7096	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7097	#endif
7098	}
7099
7100	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7101	if ( fBlockSti
7102	\|\| fBlockMovSS)
7103	{
7104	if ( !pVCpu->hm.s.fSingleInstruction
7105	&& !DBGFIsStepping(pVCpu))
7106	{
7107	/*
7108	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7109	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7110	* See Intel spec. 27.3.4 "Saving Non-Register State".
7111	*/
7112	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7113	AssertRCReturn(rc2, rc2);
7114	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7115	}
7116	else if (pMixedCtx->eflags.Bits.u1TF)
7117	{
7118	/*
7119	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7120	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7121	*/
7122	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7123	uIntrState = 0;
7124	}
7125	}
7126
7127	/*
7128	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7129	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7130	*/
7131	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7132	AssertRC(rc2);
7133
7134	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7135	NOREF(fBlockMovSS); NOREF(fBlockSti);
7136	return rc;
7137	}
7138
7139
7140	/**
7141	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7142	*
7143	* @param pVCpu Pointer to the VMCPU.
7144	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7145	* out-of-sync. Make sure to update the required fields
7146	* before using them.
7147	*/
7148	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7149	{
7150	NOREF(pMixedCtx);
7151	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7152	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7153	}
7154
7155
7156	/**
7157	* Injects a double-fault (#DF) exception into the VM.
7158	*
7159	* @returns VBox status code (informational status code included).
7160	* @param pVCpu Pointer to the VMCPU.
7161	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7162	* out-of-sync. Make sure to update the required fields
7163	* before using them.
7164	*/
7165	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7166	{
7167	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7168	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7169	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7170	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7171	puIntrState);
7172	}
7173
7174
7175	/**
7176	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7177	*
7178	* @param pVCpu Pointer to the VMCPU.
7179	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7180	* out-of-sync. Make sure to update the required fields
7181	* before using them.
7182	*/
7183	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7184	{
7185	NOREF(pMixedCtx);
7186	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7187	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7188	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7189	}
7190
7191
7192	/**
7193	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7194	*
7195	* @param pVCpu Pointer to the VMCPU.
7196	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7197	* out-of-sync. Make sure to update the required fields
7198	* before using them.
7199	* @param cbInstr The value of RIP that is to be pushed on the guest
7200	* stack.
7201	*/
7202	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7203	{
7204	NOREF(pMixedCtx);
7205	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7206	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7207	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7208	}
7209
7210
7211	/**
7212	* Injects a general-protection (#GP) fault into the VM.
7213	*
7214	* @returns VBox status code (informational status code included).
7215	* @param pVCpu Pointer to the VMCPU.
7216	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7217	* out-of-sync. Make sure to update the required fields
7218	* before using them.
7219	* @param u32ErrorCode The error code associated with the #GP.
7220	*/
7221	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7222	uint32_t *puIntrState)
7223	{
7224	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7225	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7226	if (fErrorCodeValid)
7227	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7228	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7229	puIntrState);
7230	}
7231
7232
7233	/**
7234	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7235	*
7236	* @param pVCpu Pointer to the VMCPU.
7237	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7238	* out-of-sync. Make sure to update the required fields
7239	* before using them.
7240	* @param uVector The software interrupt vector number.
7241	* @param cbInstr The value of RIP that is to be pushed on the guest
7242	* stack.
7243	*/
7244	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7245	{
7246	NOREF(pMixedCtx);
7247	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7248	if ( uVector == X86_XCPT_BP
7249	\|\| uVector == X86_XCPT_OF)
7250	{
7251	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7252	}
7253	else
7254	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7255	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7256	}
7257
7258
7259	/**
7260	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7261	* stack.
7262	*
7263	* @returns VBox status code (information status code included).
7264	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7265	* @param pVM Pointer to the VM.
7266	* @param pMixedCtx Pointer to the guest-CPU context.
7267	* @param uValue The value to push to the guest stack.
7268	*/
7269	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7270	{
7271	/*
7272	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7273	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7274	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7275	*/
7276	if (pMixedCtx->sp == 1)
7277	return VINF_EM_RESET;
7278	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7279	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7280	AssertRCReturn(rc, rc);
7281	return rc;
7282	}
7283
7284
7285	/**
7286	* Injects an event into the guest upon VM-entry by updating the relevant fields
7287	* in the VM-entry area in the VMCS.
7288	*
7289	* @returns VBox status code (informational error codes included).
7290	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7291	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7292	*
7293	* @param pVCpu Pointer to the VMCPU.
7294	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7295	* be out-of-sync. Make sure to update the required
7296	* fields before using them.
7297	* @param u64IntInfo The VM-entry interruption-information field.
7298	* @param cbInstr The VM-entry instruction length in bytes (for
7299	* software interrupts, exceptions and privileged
7300	* software exceptions).
7301	* @param u32ErrCode The VM-entry exception error code.
7302	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7303	* @param puIntrState Pointer to the current guest interruptibility-state.
7304	* This interruptibility-state will be updated if
7305	* necessary. This cannot not be NULL.
7306	*
7307	* @remarks Requires CR0!
7308	* @remarks No-long-jump zone!!!
7309	*/
7310	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7311	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7312	{
7313	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7314	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7315	Assert(puIntrState);
7316	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7317
7318	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7319	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7320
7321	#ifdef VBOX_STRICT
7322	/* Validate the error-code-valid bit for hardware exceptions. */
7323	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7324	{
7325	switch (uVector)
7326	{
7327	case X86_XCPT_PF:
7328	case X86_XCPT_DF:
7329	case X86_XCPT_TS:
7330	case X86_XCPT_NP:
7331	case X86_XCPT_SS:
7332	case X86_XCPT_GP:
7333	case X86_XCPT_AC:
7334	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7335	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7336	/* fallthru */
7337	default:
7338	break;
7339	}
7340	}
7341	#endif
7342
7343	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7344	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7345	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7346
7347	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7348
7349	/* We require CR0 to check if the guest is in real-mode. */
7350	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7351	AssertRCReturn(rc, rc);
7352
7353	/*
7354	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7355	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7356	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7357	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7358	*/
7359	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7360	{
7361	PVM pVM = pVCpu->CTX_SUFF(pVM);
7362	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7363	{
7364	Assert(PDMVmmDevHeapIsEnabled(pVM));
7365	Assert(pVM->hm.s.vmx.pRealModeTSS);
7366
7367	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7368	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7369	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7370	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7371	AssertRCReturn(rc, rc);
7372	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7373
7374	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7375	const size_t cbIdtEntry = sizeof(X86IDTR16);
7376	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7377	{
7378	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7379	if (uVector == X86_XCPT_DF)
7380	return VINF_EM_RESET;
7381	else if (uVector == X86_XCPT_GP)
7382	{
7383	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7384	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7385	}
7386
7387	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7388	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7389	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7390	}
7391
7392	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7393	uint16_t uGuestIp = pMixedCtx->ip;
7394	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7395	{
7396	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7397	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7398	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7399	}
7400	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7401	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7402
7403	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7404	X86IDTR16 IdtEntry;
7405	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7406	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7407	AssertRCReturn(rc, rc);
7408
7409	/* Construct the stack frame for the interrupt/exception handler. */
7410	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7411	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7412	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7413	AssertRCReturn(rc, rc);
7414
7415	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7416	if (rc == VINF_SUCCESS)
7417	{
7418	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7419	pMixedCtx->rip = IdtEntry.offSel;
7420	pMixedCtx->cs.Sel = IdtEntry.uSel;
7421	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7422	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7423	&& uVector == X86_XCPT_PF)
7424	{
7425	pMixedCtx->cr2 = GCPtrFaultAddress;
7426	}
7427
7428	/* If any other guest-state bits are changed here, make sure to update
7429	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7430	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7431	\| HM_CHANGED_GUEST_RIP
7432	\| HM_CHANGED_GUEST_RFLAGS
7433	\| HM_CHANGED_GUEST_RSP);
7434
7435	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7436	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7437	{
7438	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7439	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7440	Log4(("Clearing inhibition due to STI.\n"));
7441	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7442	}
7443	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7444
7445	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7446	it, if we are returning to ring-3 before executing guest code. */
7447	pVCpu->hm.s.Event.fPending = false;
7448	}
7449	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7450	return rc;
7451	}
7452	else
7453	{
7454	/*
7455	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7456	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7457	*/
7458	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7459	}
7460	}
7461
7462	/* Validate. */
7463	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7464	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7465	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7466
7467	/* Inject. */
7468	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7469	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7470	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7471	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7472
7473	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7474	&& uVector == X86_XCPT_PF)
7475	{
7476	pMixedCtx->cr2 = GCPtrFaultAddress;
7477	}
7478
7479	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7480	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7481
7482	AssertRCReturn(rc, rc);
7483	return rc;
7484	}
7485
7486
7487	/**
7488	* Clears the interrupt-window exiting control in the VMCS and if necessary
7489	* clears the current event in the VMCS as well.
7490	*
7491	* @returns VBox status code.
7492	* @param pVCpu Pointer to the VMCPU.
7493	*
7494	* @remarks Use this function only to clear events that have not yet been
7495	* delivered to the guest but are injected in the VMCS!
7496	* @remarks No-long-jump zone!!!
7497	*/
7498	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7499	{
7500	int rc;
7501	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7502
7503	/* Clear interrupt-window exiting control. */
7504	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7505	{
7506	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7507	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7508	AssertRC(rc);
7509	}
7510
7511	if (!pVCpu->hm.s.Event.fPending)
7512	return;
7513
7514	#ifdef VBOX_STRICT
7515	uint32_t u32EntryInfo;
7516	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7517	AssertRC(rc);
7518	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7519	#endif
7520
7521	/* Clear the entry-interruption field (including the valid bit). */
7522	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7523	AssertRC(rc);
7524
7525	/* Clear the pending debug exception field. */
7526	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7527	AssertRC(rc);
7528	}
7529
7530
7531	/**
7532	* Enters the VT-x session.
7533	*
7534	* @returns VBox status code.
7535	* @param pVM Pointer to the VM.
7536	* @param pVCpu Pointer to the VMCPU.
7537	* @param pCpu Pointer to the CPU info struct.
7538	*/
7539	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7540	{
7541	AssertPtr(pVM);
7542	AssertPtr(pVCpu);
7543	Assert(pVM->hm.s.vmx.fSupported);
7544	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7545	NOREF(pCpu); NOREF(pVM);
7546
7547	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7548	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7549
7550	#ifdef VBOX_STRICT
7551	/* Make sure we're in VMX root mode. */
7552	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7553	if (!(u32HostCR4 & X86_CR4_VMXE))
7554	{
7555	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7556	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7557	}
7558	#endif
7559
7560	/*
7561	* Load the VCPU's VMCS as the current (and active) one.
7562	*/
7563	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7564	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7565	if (RT_FAILURE(rc))
7566	return rc;
7567
7568	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7569	pVCpu->hm.s.fLeaveDone = false;
7570	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7571
7572	return VINF_SUCCESS;
7573	}
7574
7575
7576	/**
7577	* The thread-context callback (only on platforms which support it).
7578	*
7579	* @param enmEvent The thread-context event.
7580	* @param pVCpu Pointer to the VMCPU.
7581	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7582	* @thread EMT(pVCpu)
7583	*/
7584	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7585	{
7586	NOREF(fGlobalInit);
7587
7588	switch (enmEvent)
7589	{
7590	case RTTHREADCTXEVENT_PREEMPTING:
7591	{
7592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7593	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7594	VMCPU_ASSERT_EMT(pVCpu);
7595
7596	PVM pVM = pVCpu->CTX_SUFF(pVM);
7597	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7598
7599	/* No longjmps (logger flushes, locks) in this fragile context. */
7600	VMMRZCallRing3Disable(pVCpu);
7601	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7602
7603	/*
7604	* Restore host-state (FPU, debug etc.)
7605	*/
7606	if (!pVCpu->hm.s.fLeaveDone)
7607	{
7608	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7609	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7610	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7611	pVCpu->hm.s.fLeaveDone = true;
7612	}
7613
7614	/* Leave HM context, takes care of local init (term). */
7615	int rc = HMR0LeaveCpu(pVCpu);
7616	AssertRC(rc); NOREF(rc);
7617
7618	/* Restore longjmp state. */
7619	VMMRZCallRing3Enable(pVCpu);
7620	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7621	break;
7622	}
7623
7624	case RTTHREADCTXEVENT_RESUMED:
7625	{
7626	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7627	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7628	VMCPU_ASSERT_EMT(pVCpu);
7629
7630	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7631	VMMRZCallRing3Disable(pVCpu);
7632	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7633
7634	/* Initialize the bare minimum state required for HM. This takes care of
7635	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7636	int rc = HMR0EnterCpu(pVCpu);
7637	AssertRC(rc);
7638	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7639
7640	/* Load the active VMCS as the current one. */
7641	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7642	{
7643	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7644	AssertRC(rc); NOREF(rc);
7645	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7646	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7647	}
7648	pVCpu->hm.s.fLeaveDone = false;
7649
7650	/* Restore longjmp state. */
7651	VMMRZCallRing3Enable(pVCpu);
7652	break;
7653	}
7654
7655	default:
7656	break;
7657	}
7658	}
7659
7660
7661	/**
7662	* Saves the host state in the VMCS host-state.
7663	* Sets up the VM-exit MSR-load area.
7664	*
7665	* The CPU state will be loaded from these fields on every successful VM-exit.
7666	*
7667	* @returns VBox status code.
7668	* @param pVM Pointer to the VM.
7669	* @param pVCpu Pointer to the VMCPU.
7670	*
7671	* @remarks No-long-jump zone!!!
7672	*/
7673	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7674	{
7675	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7676
7677	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7678	return VINF_SUCCESS;
7679
7680	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7681	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7682
7683	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7684	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7685
7686	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7687	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7688
7689	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7690	return rc;
7691	}
7692
7693
7694	/**
7695	* Saves the host state in the VMCS host-state.
7696	*
7697	* @returns VBox status code.
7698	* @param pVM Pointer to the VM.
7699	* @param pVCpu Pointer to the VMCPU.
7700	*
7701	* @remarks No-long-jump zone!!!
7702	*/
7703	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7704	{
7705	AssertPtr(pVM);
7706	AssertPtr(pVCpu);
7707
7708	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7709
7710	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7711	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7712	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7713	return hmR0VmxSaveHostState(pVM, pVCpu);
7714	}
7715
7716
7717	/**
7718	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7719	* loaded from these fields on every successful VM-entry.
7720	*
7721	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7722	* Sets up the VM-entry controls.
7723	* Sets up the appropriate VMX non-root function to execute guest code based on
7724	* the guest CPU mode.
7725	*
7726	* @returns VBox status code.
7727	* @param pVM Pointer to the VM.
7728	* @param pVCpu Pointer to the VMCPU.
7729	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7730	* out-of-sync. Make sure to update the required fields
7731	* before using them.
7732	*
7733	* @remarks No-long-jump zone!!!
7734	*/
7735	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7736	{
7737	AssertPtr(pVM);
7738	AssertPtr(pVCpu);
7739	AssertPtr(pMixedCtx);
7740	HMVMX_ASSERT_PREEMPT_SAFE();
7741
7742	#ifdef LOG_ENABLED
7743	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7744	* probably not initialized yet? Anyway this will do for now.
7745	*
7746	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7747	* interface and disable ring-3 calls when thread-context hooks are not
7748	* available. */
7749	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7750	VMMR0LogFlushDisable(pVCpu);
7751	#endif
7752
7753	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7754
7755	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7756
7757	/* Determine real-on-v86 mode. */
7758	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7759	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7760	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7761	{
7762	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7763	}
7764
7765	/*
7766	* Load the guest-state into the VMCS.
7767	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7768	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7769	*/
7770	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7771	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7772
7773	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7774	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7775	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7776
7777	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7778	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7779	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7780
7781	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7782	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7783
7784	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7785	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7786
7787	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7788	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7789	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7790
7791	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7792	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7793
7794	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7795	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7796
7797	/*
7798	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7799	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7800	*/
7801	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7802	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7803
7804	/* Clear any unused and reserved bits. */
7805	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7806
7807	#ifdef LOG_ENABLED
7808	/* Only reenable log-flushing if the caller has it enabled. */
7809	if (!fCallerDisabledLogFlush)
7810	VMMR0LogFlushEnable(pVCpu);
7811	#endif
7812
7813	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7814	return rc;
7815	}
7816
7817
7818	/**
7819	* Loads the state shared between the host and guest into the VMCS.
7820	*
7821	* @param pVM Pointer to the VM.
7822	* @param pVCpu Pointer to the VMCPU.
7823	* @param pCtx Pointer to the guest-CPU context.
7824	*
7825	* @remarks No-long-jump zone!!!
7826	*/
7827	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7828	{
7829	NOREF(pVM);
7830
7831	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7832	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7833
7834	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7835	{
7836	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7837	AssertRC(rc);
7838	}
7839
7840	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7841	{
7842	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7843	AssertRC(rc);
7844
7845	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7846	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7847	{
7848	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7849	AssertRC(rc);
7850	}
7851	}
7852
7853	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7854	{
7855	#if HC_ARCH_BITS == 64
7856	if (pVM->hm.s.fAllow64BitGuests)
7857	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7858	#endif
7859	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7860	}
7861
7862	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7863	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7864	}
7865
7866
7867	/**
7868	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7869	*
7870	* @param pVM Pointer to the VM.
7871	* @param pVCpu Pointer to the VMCPU.
7872	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7873	* out-of-sync. Make sure to update the required fields
7874	* before using them.
7875	*/
7876	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7877	{
7878	HMVMX_ASSERT_PREEMPT_SAFE();
7879
7880	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7881	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7882	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7883	#endif
7884
7885	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7886	{
7887	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7888	AssertRC(rc);
7889	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7890	}
7891	else if (HMCPU_CF_VALUE(pVCpu))
7892	{
7893	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7894	AssertRC(rc);
7895	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7896	}
7897
7898	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7899	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7900	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7901	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7902
7903	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7904	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7905	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7906	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7907	#endif
7908	}
7909
7910
7911	/**
7912	* Does the preparations before executing guest code in VT-x.
7913	*
7914	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7915	* recompiler. We must be cautious what we do here regarding committing
7916	* guest-state information into the VMCS assuming we assuredly execute the
7917	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7918	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7919	* so that the recompiler can (and should) use them when it resumes guest
7920	* execution. Otherwise such operations must be done when we can no longer
7921	* exit to ring-3.
7922	*
7923	* @returns Strict VBox status code.
7924	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7925	* have been disabled.
7926	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7927	* double-fault into the guest.
7928	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7929	*
7930	* @param pVM Pointer to the VM.
7931	* @param pVCpu Pointer to the VMCPU.
7932	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7933	* out-of-sync. Make sure to update the required fields
7934	* before using them.
7935	* @param pVmxTransient Pointer to the VMX transient structure.
7936	*/
7937	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7938	{
7939	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7940
7941	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7942	PGMRZDynMapFlushAutoSet(pVCpu);
7943	#endif
7944
7945	/* Check force flag actions that might require us to go back to ring-3. */
7946	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7947	if (rc != VINF_SUCCESS)
7948	return rc;
7949
7950	#ifndef IEM_VERIFICATION_MODE_FULL
7951	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7952	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7953	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7954	{
7955	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7956	RTGCPHYS GCPhysApicBase;
7957	GCPhysApicBase = pMixedCtx->msrApicBase;
7958	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7959
7960	/* Unalias any existing mapping. */
7961	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7962	AssertRCReturn(rc, rc);
7963
7964	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7965	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7966	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7967	AssertRCReturn(rc, rc);
7968
7969	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7970	}
7971	#endif /* !IEM_VERIFICATION_MODE_FULL */
7972
7973	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7974	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7975
7976	/*
7977	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7978	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7979	*/
7980	if (TRPMHasTrap(pVCpu))
7981	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7982	else if (!pVCpu->hm.s.Event.fPending)
7983	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7984
7985	/*
7986	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7987	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7988	*/
7989	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7990	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7991	{
7992	Assert(rc == VINF_EM_RESET);
7993	return rc;
7994	}
7995
7996	/*
7997	* No longjmps to ring-3 from this point on!!!
7998	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7999	* This also disables flushing of the R0-logger instance (if any).
8000	*/
8001	VMMRZCallRing3Disable(pVCpu);
8002
8003	/*
8004	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8005	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8006	*
8007	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8008	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8009	*
8010	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8011	* executing guest code.
8012	*/
8013	pVmxTransient->uEflags = ASMIntDisableFlags();
8014	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8015	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8016	{
8017	hmR0VmxClearEventVmcs(pVCpu);
8018	ASMSetFlags(pVmxTransient->uEflags);
8019	VMMRZCallRing3Enable(pVCpu);
8020	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8021	return VINF_EM_RAW_TO_R3;
8022	}
8023	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8024	{
8025	hmR0VmxClearEventVmcs(pVCpu);
8026	ASMSetFlags(pVmxTransient->uEflags);
8027	VMMRZCallRing3Enable(pVCpu);
8028	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8029	return VINF_EM_RAW_INTERRUPT;
8030	}
8031
8032	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8033	pVCpu->hm.s.Event.fPending = false;
8034
8035	return VINF_SUCCESS;
8036	}
8037
8038
8039	/**
8040	* Prepares to run guest code in VT-x and we've committed to doing so. This
8041	* means there is no backing out to ring-3 or anywhere else at this
8042	* point.
8043	*
8044	* @param pVM Pointer to the VM.
8045	* @param pVCpu Pointer to the VMCPU.
8046	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8047	* out-of-sync. Make sure to update the required fields
8048	* before using them.
8049	* @param pVmxTransient Pointer to the VMX transient structure.
8050	*
8051	* @remarks Called with preemption disabled.
8052	* @remarks No-long-jump zone!!!
8053	*/
8054	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8055	{
8056	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8057	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8058	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8059
8060	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8061	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8062
8063	/*
8064	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8065	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8066	* Reload only the necessary state, the assertion will catch if other parts of the code
8067	* change.
8068	*/
8069	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8070	{
8071	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8072	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8073	}
8074
8075	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8076	if (!CPUMIsGuestFPUStateActive(pVCpu))
8077	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8078	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8079	#endif
8080
8081	if ( pVCpu->hm.s.fUseGuestFpu
8082	&& !CPUMIsGuestFPUStateActive(pVCpu))
8083	{
8084	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8085	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8086	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8087	}
8088
8089	/*
8090	* The host MSR values the very first time around won't be updated, so we need to
8091	* fill those values in. Subsequently, it's updated as part of the host state.
8092	*/
8093	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8094	&& pVCpu->hm.s.vmx.cMsrs > 0)
8095	{
8096	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8097	}
8098
8099	/*
8100	* Load the host state bits as we may've been preempted (only happens when
8101	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8102	*/
8103	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8104	{
8105	/* This ASSUMES that pfnStartVM has been set up already. */
8106	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8107	AssertRC(rc);
8108	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8109	}
8110	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8111
8112	/*
8113	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8114	*/
8115	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8116	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8117	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8118
8119	/* Store status of the shared guest-host state at the time of VM-entry. */
8120	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8121	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8122	{
8123	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8124	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8125	}
8126	else
8127	#endif
8128	{
8129	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8130	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8131	}
8132	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8133
8134	/*
8135	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8136	*/
8137	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8138	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8139
8140	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8141	RTCPUID idCurrentCpu = pCpu->idCpu;
8142	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8143	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8144	{
8145	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8146	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8147	}
8148
8149	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8150	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8151	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8152	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8153
8154	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8155
8156	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8157	to start executing. */
8158
8159	/*
8160	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8161	*/
8162	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8163	{
8164	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8165	{
8166	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8167	AssertRC(rc2);
8168	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8169	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */);
8170	}
8171	else
8172	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8173	}
8174	#ifdef VBOX_STRICT
8175	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8176	#endif
8177	}
8178
8179
8180	/**
8181	* Performs some essential restoration of state after running guest code in
8182	* VT-x.
8183	*
8184	* @param pVM Pointer to the VM.
8185	* @param pVCpu Pointer to the VMCPU.
8186	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8187	* out-of-sync. Make sure to update the required fields
8188	* before using them.
8189	* @param pVmxTransient Pointer to the VMX transient structure.
8190	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8191	*
8192	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8193	*
8194	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8195	* unconditionally when it is safe to do so.
8196	*/
8197	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8198	{
8199	NOREF(pVM);
8200
8201	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8202
8203	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8204	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8205	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8206	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8207	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8208
8209	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8210	{
8211	/** @todo Find a way to fix hardcoding a guestimate. */
8212	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8213	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8214	}
8215
8216	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8217	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8218	Assert(!(ASMGetFlags() & X86_EFL_IF));
8219	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8220
8221	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8222	if (CPUMIsGuestFPUStateActive(pVCpu))
8223	{
8224	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8225	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8226	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8227	}
8228	#endif
8229
8230	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8231	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8232	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8233	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8234
8235	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8236	uint32_t uExitReason;
8237	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8238	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8239	AssertRC(rc);
8240	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8241	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8242
8243	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8244	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8245	{
8246	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8247	pVmxTransient->fVMEntryFailed));
8248	return;
8249	}
8250
8251	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8252	{
8253	/* Update the guest interruptibility-state from the VMCS. */
8254	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8255	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8256	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8257	AssertRC(rc);
8258	#endif
8259	/*
8260	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8261	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8262	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8263	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8264	*/
8265	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8266	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8267	{
8268	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8269	AssertRC(rc);
8270	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8271	}
8272	}
8273	}
8274
8275
8276
8277	/**
8278	* Runs the guest code using VT-x the normal way.
8279	*
8280	* @returns VBox status code.
8281	* @param pVM Pointer to the VM.
8282	* @param pVCpu Pointer to the VMCPU.
8283	* @param pCtx Pointer to the guest-CPU context.
8284	*
8285	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8286	*/
8287	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8288	{
8289	VMXTRANSIENT VmxTransient;
8290	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8291	int rc = VERR_INTERNAL_ERROR_5;
8292	uint32_t cLoops = 0;
8293
8294	for (;; cLoops++)
8295	{
8296	Assert(!HMR0SuspendPending());
8297	HMVMX_ASSERT_CPU_SAFE();
8298
8299	/* Preparatory work for running guest code, this may force us to return
8300	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8301	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8302	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8303	if (rc != VINF_SUCCESS)
8304	break;
8305
8306	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8307	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8308	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8309
8310	/* Restore any residual host-state and save any bits shared between host
8311	and guest into the guest-CPU state. Re-enables interrupts! */
8312	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8313
8314	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8315	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8316	{
8317	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8318	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8319	return rc;
8320	}
8321
8322	/* Handle the VM-exit. */
8323	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8325	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8326	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8327	HMVMX_START_EXIT_DISPATCH_PROF();
8328	#ifdef HMVMX_USE_FUNCTION_TABLE
8329	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8330	#else
8331	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8332	#endif
8333	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8334	if (rc != VINF_SUCCESS)
8335	break;
8336	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8337	{
8338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8339	rc = VINF_EM_RAW_INTERRUPT;
8340	break;
8341	}
8342	}
8343
8344	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8345	return rc;
8346	}
8347
8348
8349	/**
8350	* Single steps guest code using VT-x.
8351	*
8352	* @returns VBox status code.
8353	* @param pVM Pointer to the VM.
8354	* @param pVCpu Pointer to the VMCPU.
8355	* @param pCtx Pointer to the guest-CPU context.
8356	*
8357	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8358	*/
8359	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8360	{
8361	VMXTRANSIENT VmxTransient;
8362	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8363	int rc = VERR_INTERNAL_ERROR_5;
8364	uint32_t cLoops = 0;
8365	uint16_t uCsStart = pCtx->cs.Sel;
8366	uint64_t uRipStart = pCtx->rip;
8367
8368	for (;; cLoops++)
8369	{
8370	Assert(!HMR0SuspendPending());
8371	HMVMX_ASSERT_CPU_SAFE();
8372
8373	/* Preparatory work for running guest code, this may force us to return
8374	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8375	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8376	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8377	if (rc != VINF_SUCCESS)
8378	break;
8379
8380	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8381	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8382	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8383
8384	/* Restore any residual host-state and save any bits shared between host
8385	and guest into the guest-CPU state. Re-enables interrupts! */
8386	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8387
8388	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8389	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8390	{
8391	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8392	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8393	return rc;
8394	}
8395
8396	/* Handle the VM-exit. */
8397	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8398	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8399	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8400	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8401	HMVMX_START_EXIT_DISPATCH_PROF();
8402	#ifdef HMVMX_USE_FUNCTION_TABLE
8403	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8404	#else
8405	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8406	#endif
8407	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8408	if (rc != VINF_SUCCESS)
8409	break;
8410	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8411	{
8412	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8413	rc = VINF_EM_RAW_INTERRUPT;
8414	break;
8415	}
8416
8417	/*
8418	* Did the RIP change, if so, consider it a single step.
8419	* Otherwise, make sure one of the TFs gets set.
8420	*/
8421	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8422	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8423	AssertRCReturn(rc2, rc2);
8424	if ( pCtx->rip != uRipStart
8425	\|\| pCtx->cs.Sel != uCsStart)
8426	{
8427	rc = VINF_EM_DBG_STEPPED;
8428	break;
8429	}
8430	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8431	}
8432
8433	/*
8434	* Clear the X86_EFL_TF if necessary.
8435	*/
8436	if (pVCpu->hm.s.fClearTrapFlag)
8437	{
8438	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8439	AssertRCReturn(rc2, rc2);
8440	pVCpu->hm.s.fClearTrapFlag = false;
8441	pCtx->eflags.Bits.u1TF = 0;
8442	}
8443	/** @todo there seems to be issues with the resume flag when the monitor trap
8444	* flag is pending without being used. Seen early in bios init when
8445	* accessing APIC page in prot mode. */
8446
8447	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8448	return rc;
8449	}
8450
8451
8452	/**
8453	* Runs the guest code using VT-x.
8454	*
8455	* @returns VBox status code.
8456	* @param pVM Pointer to the VM.
8457	* @param pVCpu Pointer to the VMCPU.
8458	* @param pCtx Pointer to the guest-CPU context.
8459	*/
8460	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8461	{
8462	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8463	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8464	HMVMX_ASSERT_PREEMPT_SAFE();
8465
8466	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8467
8468	int rc;
8469	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8470	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8471	else
8472	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8473
8474	if (rc == VERR_EM_INTERPRETER)
8475	rc = VINF_EM_RAW_EMULATE_INSTR;
8476	else if (rc == VINF_EM_RESET)
8477	rc = VINF_EM_TRIPLE_FAULT;
8478
8479	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8480	if (RT_FAILURE(rc2))
8481	{
8482	pVCpu->hm.s.u32HMError = rc;
8483	rc = rc2;
8484	}
8485	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8486	return rc;
8487	}
8488
8489
8490	#ifndef HMVMX_USE_FUNCTION_TABLE
8491	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8492	{
8493	#ifdef DEBUG_ramshankar
8494	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8495	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8496	#endif
8497	int rc;
8498	switch (rcReason)
8499	{
8500	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8508	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8509	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8510	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8511	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8512	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8513	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8514	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8515	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8516	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8517	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8518	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8519	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8520	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8521	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8522	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8523	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8524	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8525	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8526	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8527	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8528	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8529	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8530	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8531	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8532	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8533
8534	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8535	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8536	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8537	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8538	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8539	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8540	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8541	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8542	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8543
8544	case VMX_EXIT_VMCALL:
8545	case VMX_EXIT_VMCLEAR:
8546	case VMX_EXIT_VMLAUNCH:
8547	case VMX_EXIT_VMPTRLD:
8548	case VMX_EXIT_VMPTRST:
8549	case VMX_EXIT_VMREAD:
8550	case VMX_EXIT_VMRESUME:
8551	case VMX_EXIT_VMWRITE:
8552	case VMX_EXIT_VMXOFF:
8553	case VMX_EXIT_VMXON:
8554	case VMX_EXIT_INVEPT:
8555	case VMX_EXIT_INVVPID:
8556	case VMX_EXIT_VMFUNC:
8557	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8558	break;
8559	default:
8560	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8561	break;
8562	}
8563	return rc;
8564	}
8565	#endif
8566
8567	#ifdef DEBUG
8568	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8569	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8570	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8571
8572	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8573	do \
8574	{ \
8575	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8576	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8577	} while (0)
8578
8579	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8580	do { \
8581	AssertPtr(pVCpu); \
8582	AssertPtr(pMixedCtx); \
8583	AssertPtr(pVmxTransient); \
8584	Assert(pVmxTransient->fVMEntryFailed == false); \
8585	Assert(ASMIntAreEnabled()); \
8586	HMVMX_ASSERT_PREEMPT_SAFE(); \
8587	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8588	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8589	HMVMX_ASSERT_PREEMPT_SAFE(); \
8590	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8591	HMVMX_ASSERT_PREEMPT_CPUID(); \
8592	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8593	} while (0)
8594
8595	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8596	do { \
8597	Log4Func(("\n")); \
8598	} while (0)
8599	#else /* Release builds */
8600	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8601	do { \
8602	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8603	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8604	} while (0)
8605	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8606	#endif
8607
8608
8609	/**
8610	* Advances the guest RIP after reading it from the VMCS.
8611	*
8612	* @returns VBox status code.
8613	* @param pVCpu Pointer to the VMCPU.
8614	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8615	* out-of-sync. Make sure to update the required fields
8616	* before using them.
8617	* @param pVmxTransient Pointer to the VMX transient structure.
8618	*
8619	* @remarks No-long-jump zone!!!
8620	*/
8621	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8622	{
8623	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8624	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8625	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8626	AssertRCReturn(rc, rc);
8627
8628	pMixedCtx->rip += pVmxTransient->cbInstr;
8629	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8630
8631	/*
8632	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8633	* pending debug exception field as it takes care of priority of events.
8634	*
8635	* See Intel spec. 32.2.1 "Debug Exceptions".
8636	*/
8637	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8638
8639	return rc;
8640	}
8641
8642
8643	/**
8644	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8645	* and update error record fields accordingly.
8646	*
8647	* @return VMX_IGS_* return codes.
8648	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8649	* wrong with the guest state.
8650	*
8651	* @param pVM Pointer to the VM.
8652	* @param pVCpu Pointer to the VMCPU.
8653	* @param pCtx Pointer to the guest-CPU state.
8654	*/
8655	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8656	{
8657	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8658	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8659	uError = (err); \
8660	break; \
8661	} else do { } while (0)
8662
8663	int rc;
8664	uint32_t uError = VMX_IGS_ERROR;
8665	uint32_t u32Val;
8666	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8667
8668	do
8669	{
8670	/*
8671	* CR0.
8672	*/
8673	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8674	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8675	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8676	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8677	if (fUnrestrictedGuest)
8678	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8679
8680	uint32_t u32GuestCR0;
8681	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8682	AssertRCBreak(rc);
8683	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8684	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8685	if ( !fUnrestrictedGuest
8686	&& (u32GuestCR0 & X86_CR0_PG)
8687	&& !(u32GuestCR0 & X86_CR0_PE))
8688	{
8689	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8690	}
8691
8692	/*
8693	* CR4.
8694	*/
8695	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8696	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8697
8698	uint32_t u32GuestCR4;
8699	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8700	AssertRCBreak(rc);
8701	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8702	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8703
8704	/*
8705	* IA32_DEBUGCTL MSR.
8706	*/
8707	uint64_t u64Val;
8708	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8709	AssertRCBreak(rc);
8710	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8711	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8712	{
8713	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8714	}
8715	uint64_t u64DebugCtlMsr = u64Val;
8716
8717	#ifdef VBOX_STRICT
8718	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8719	AssertRCBreak(rc);
8720	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8721	#endif
8722	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8723
8724	/*
8725	* RIP and RFLAGS.
8726	*/
8727	uint32_t u32Eflags;
8728	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8729	if (HMVMX_IS_64BIT_HOST_MODE())
8730	{
8731	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8732	AssertRCBreak(rc);
8733	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8734	if ( !fLongModeGuest
8735	\|\| !pCtx->cs.Attr.n.u1Long)
8736	{
8737	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8738	}
8739	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8740	* must be identical if the "IA32e mode guest" VM-entry control is 1
8741	* and CS.L is 1. No check applies if the CPU supports 64
8742	* linear-address bits. */
8743
8744	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8745	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8746	AssertRCBreak(rc);
8747	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8748	VMX_IGS_RFLAGS_RESERVED);
8749	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8750	u32Eflags = u64Val;
8751	}
8752	else
8753	#endif
8754	{
8755	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8756	AssertRCBreak(rc);
8757	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8758	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8759	}
8760
8761	if ( fLongModeGuest
8762	\|\| ( fUnrestrictedGuest
8763	&& !(u32GuestCR0 & X86_CR0_PE)))
8764	{
8765	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8766	}
8767
8768	uint32_t u32EntryInfo;
8769	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8770	AssertRCBreak(rc);
8771	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8772	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8773	{
8774	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8775	}
8776
8777	/*
8778	* 64-bit checks.
8779	*/
8780	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8781	if (HMVMX_IS_64BIT_HOST_MODE())
8782	{
8783	if ( fLongModeGuest
8784	&& !fUnrestrictedGuest)
8785	{
8786	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8787	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8788	}
8789
8790	if ( !fLongModeGuest
8791	&& (u32GuestCR4 & X86_CR4_PCIDE))
8792	{
8793	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8794	}
8795
8796	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8797	* 51:32 beyond the processor's physical-address width are 0. */
8798
8799	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8800	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8801	{
8802	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8803	}
8804
8805	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8806	AssertRCBreak(rc);
8807	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8808
8809	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8810	AssertRCBreak(rc);
8811	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8812	}
8813	#endif
8814
8815	/*
8816	* PERF_GLOBAL MSR.
8817	*/
8818	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8819	{
8820	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8821	AssertRCBreak(rc);
8822	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8823	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8824	}
8825
8826	/*
8827	* PAT MSR.
8828	*/
8829	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8830	{
8831	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8832	AssertRCBreak(rc);
8833	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8834	for (unsigned i = 0; i < 8; i++)
8835	{
8836	uint8_t u8Val = (u64Val & 0x7);
8837	if ( u8Val != 0 /* UC */
8838	\|\| u8Val != 1 /* WC */
8839	\|\| u8Val != 4 /* WT */
8840	\|\| u8Val != 5 /* WP */
8841	\|\| u8Val != 6 /* WB */
8842	\|\| u8Val != 7 /* UC- */)
8843	{
8844	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8845	}
8846	u64Val >>= 3;
8847	}
8848	}
8849
8850	/*
8851	* EFER MSR.
8852	*/
8853	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8854	{
8855	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8856	AssertRCBreak(rc);
8857	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8858	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8859	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8860	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8861	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8862	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8863	}
8864
8865	/*
8866	* Segment registers.
8867	*/
8868	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8869	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8870	if (!(u32Eflags & X86_EFL_VM))
8871	{
8872	/* CS */
8873	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8874	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8875	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8876	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8877	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8878	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8879	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8880	/* CS cannot be loaded with NULL in protected mode. */
8881	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8882	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8883	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8884	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8885	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8886	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8887	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8888	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8889	else
8890	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8891
8892	/* SS */
8893	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8894	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8895	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8896	if ( !(pCtx->cr0 & X86_CR0_PE)
8897	\|\| pCtx->cs.Attr.n.u4Type == 3)
8898	{
8899	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8900	}
8901	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8902	{
8903	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8904	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8905	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8906	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8907	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8908	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8909	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8910	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8911	}
8912
8913	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8914	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8915	{
8916	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8917	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8918	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8919	\|\| pCtx->ds.Attr.n.u4Type > 11
8920	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8921	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8922	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8923	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8924	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8925	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8926	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8927	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8928	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8929	}
8930	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8931	{
8932	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8933	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8934	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8935	\|\| pCtx->es.Attr.n.u4Type > 11
8936	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8937	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8938	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8939	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8940	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8941	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8942	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8943	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8944	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8945	}
8946	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8947	{
8948	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8949	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8950	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8951	\|\| pCtx->fs.Attr.n.u4Type > 11
8952	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8953	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8954	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8955	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8956	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8957	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8958	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8959	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8960	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8961	}
8962	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8963	{
8964	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8965	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8966	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8967	\|\| pCtx->gs.Attr.n.u4Type > 11
8968	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8969	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8970	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8971	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8972	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8973	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8974	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8975	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8976	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8977	}
8978	/* 64-bit capable CPUs. */
8979	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8980	if (HMVMX_IS_64BIT_HOST_MODE())
8981	{
8982	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8983	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8984	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8985	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8986	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8987	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8988	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8989	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8990	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8991	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8992	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8993	}
8994	#endif
8995	}
8996	else
8997	{
8998	/* V86 mode checks. */
8999	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9000	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9001	{
9002	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9003	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9004	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9005	}
9006	else
9007	{
9008	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9009	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9010	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9011	}
9012
9013	/* CS */
9014	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9015	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9016	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9017	/* SS */
9018	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9019	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9020	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9021	/* DS */
9022	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9023	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9024	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9025	/* ES */
9026	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9027	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9028	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9029	/* FS */
9030	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9031	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9032	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9033	/* GS */
9034	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9035	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9036	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9037	/* 64-bit capable CPUs. */
9038	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9039	if (HMVMX_IS_64BIT_HOST_MODE())
9040	{
9041	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9042	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9043	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9044	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9045	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9046	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9047	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9048	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9049	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9050	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9051	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9052	}
9053	#endif
9054	}
9055
9056	/*
9057	* TR.
9058	*/
9059	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9060	/* 64-bit capable CPUs. */
9061	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9062	if (HMVMX_IS_64BIT_HOST_MODE())
9063	{
9064	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9065	}
9066	#endif
9067	if (fLongModeGuest)
9068	{
9069	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9070	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9071	}
9072	else
9073	{
9074	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9075	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9076	VMX_IGS_TR_ATTR_TYPE_INVALID);
9077	}
9078	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9079	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9080	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9081	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9082	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9083	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9084	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9085	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9086
9087	/*
9088	* GDTR and IDTR.
9089	*/
9090	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9091	if (HMVMX_IS_64BIT_HOST_MODE())
9092	{
9093	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9094	AssertRCBreak(rc);
9095	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9096
9097	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9098	AssertRCBreak(rc);
9099	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9100	}
9101	#endif
9102
9103	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9104	AssertRCBreak(rc);
9105	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9106
9107	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9108	AssertRCBreak(rc);
9109	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9110
9111	/*
9112	* Guest Non-Register State.
9113	*/
9114	/* Activity State. */
9115	uint32_t u32ActivityState;
9116	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9117	AssertRCBreak(rc);
9118	HMVMX_CHECK_BREAK( !u32ActivityState
9119	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9120	VMX_IGS_ACTIVITY_STATE_INVALID);
9121	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9122	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9123	uint32_t u32IntrState;
9124	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9125	AssertRCBreak(rc);
9126	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9127	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9128	{
9129	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9130	}
9131
9132	/** @todo Activity state and injecting interrupts. Left as a todo since we
9133	* currently don't use activity states but ACTIVE. */
9134
9135	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9136	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9137
9138	/* Guest interruptibility-state. */
9139	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9140	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9141	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9142	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9143	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9144	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9145	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9146	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9147	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9148	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9149	{
9150	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9151	{
9152	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9153	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9154	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9155	}
9156	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9157	{
9158	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9159	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9160	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9161	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9162	}
9163	}
9164	/** @todo Assumes the processor is not in SMM. */
9165	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9166	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9167	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9168	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9169	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9170	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9171	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9172	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9173	{
9174	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9175	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9176	}
9177
9178	/* Pending debug exceptions. */
9179	if (HMVMX_IS_64BIT_HOST_MODE())
9180	{
9181	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9182	AssertRCBreak(rc);
9183	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9184	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9185	u32Val = u64Val; /* For pending debug exceptions checks below. */
9186	}
9187	else
9188	{
9189	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9190	AssertRCBreak(rc);
9191	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9192	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9193	}
9194
9195	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9196	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9197	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9198	{
9199	if ( (u32Eflags & X86_EFL_TF)
9200	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9201	{
9202	/* Bit 14 is PendingDebug.BS. */
9203	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9204	}
9205	if ( !(u32Eflags & X86_EFL_TF)
9206	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9207	{
9208	/* Bit 14 is PendingDebug.BS. */
9209	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9210	}
9211	}
9212
9213	/* VMCS link pointer. */
9214	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9215	AssertRCBreak(rc);
9216	if (u64Val != UINT64_C(0xffffffffffffffff))
9217	{
9218	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9219	/** @todo Bits beyond the processor's physical-address width MBZ. */
9220	/** @todo 32-bit located in memory referenced by value of this field (as a
9221	* physical address) must contain the processor's VMCS revision ID. */
9222	/** @todo SMM checks. */
9223	}
9224
9225	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9226	* not using Nested Paging? */
9227	if ( pVM->hm.s.fNestedPaging
9228	&& !fLongModeGuest
9229	&& CPUMIsGuestInPAEModeEx(pCtx))
9230	{
9231	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9232	AssertRCBreak(rc);
9233	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9234
9235	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9236	AssertRCBreak(rc);
9237	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9238
9239	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9240	AssertRCBreak(rc);
9241	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9242
9243	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9244	AssertRCBreak(rc);
9245	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9246	}
9247
9248	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9249	if (uError == VMX_IGS_ERROR)
9250	uError = VMX_IGS_REASON_NOT_FOUND;
9251	} while (0);
9252
9253	pVCpu->hm.s.u32HMError = uError;
9254	return uError;
9255
9256	#undef HMVMX_ERROR_BREAK
9257	#undef HMVMX_CHECK_BREAK
9258	}
9259
9260	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9261	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9262	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9263
9264	/** @name VM-exit handlers.
9265	* @{
9266	*/
9267
9268	/**
9269	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9270	*/
9271	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9272	{
9273	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9274	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9275	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9276	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9277	return VINF_SUCCESS;
9278	return VINF_EM_RAW_INTERRUPT;
9279	}
9280
9281
9282	/**
9283	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9284	*/
9285	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9286	{
9287	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9288	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9289
9290	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9291	AssertRCReturn(rc, rc);
9292
9293	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9294	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9295	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9296	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9297
9298	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9299	{
9300	/*
9301	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9302	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9303	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9304	*
9305	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9306	*/
9307	VMXDispatchHostNmi();
9308	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9309	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9310	return VINF_SUCCESS;
9311	}
9312
9313	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9314	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9315	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9316	{
9317	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9318	return VINF_SUCCESS;
9319	}
9320	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9321	{
9322	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9323	return rc;
9324	}
9325
9326	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9327	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9328	switch (uIntType)
9329	{
9330	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9331	Assert(uVector == X86_XCPT_DB);
9332	/* no break */
9333	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9334	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9335	/* no break */
9336	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9337	{
9338	switch (uVector)
9339	{
9340	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9341	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9342	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9343	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9344	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9345	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9346	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9347	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9348	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9349	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9350	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9351	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9352	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9353	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9354	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9355	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9356	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9357	#endif
9358	default:
9359	{
9360	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9361	AssertRCReturn(rc, rc);
9362
9363	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9364	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9365	{
9366	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9367	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9368	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9369
9370	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9371	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9372	AssertRCReturn(rc, rc);
9373	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9374	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9375	0 /* GCPtrFaultAddress */);
9376	AssertRCReturn(rc, rc);
9377	}
9378	else
9379	{
9380	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9381	pVCpu->hm.s.u32HMError = uVector;
9382	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9383	}
9384	break;
9385	}
9386	}
9387	break;
9388	}
9389
9390	default:
9391	{
9392	pVCpu->hm.s.u32HMError = uExitIntInfo;
9393	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9394	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9395	break;
9396	}
9397	}
9398	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9399	return rc;
9400	}
9401
9402
9403	/**
9404	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9405	*/
9406	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9407	{
9408	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9409
9410	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9411	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9412	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9413	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9414	AssertRCReturn(rc, rc);
9415
9416	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9418	return VINF_SUCCESS;
9419	}
9420
9421
9422	/**
9423	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9424	*/
9425	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9426	{
9427	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9428	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9429	HMVMX_RETURN_UNEXPECTED_EXIT();
9430	}
9431
9432
9433	/**
9434	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9435	*/
9436	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9437	{
9438	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9439	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9440	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9441	}
9442
9443
9444	/**
9445	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9446	*/
9447	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9448	{
9449	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9450	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9451	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9452	}
9453
9454
9455	/**
9456	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9457	*/
9458	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9459	{
9460	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9461	PVM pVM = pVCpu->CTX_SUFF(pVM);
9462	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9463	if (RT_LIKELY(rc == VINF_SUCCESS))
9464	{
9465	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9466	Assert(pVmxTransient->cbInstr == 2);
9467	}
9468	else
9469	{
9470	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9471	rc = VERR_EM_INTERPRETER;
9472	}
9473	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9474	return rc;
9475	}
9476
9477
9478	/**
9479	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9480	*/
9481	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9482	{
9483	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9484	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9485	AssertRCReturn(rc, rc);
9486
9487	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9488	return VINF_EM_RAW_EMULATE_INSTR;
9489
9490	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9491	HMVMX_RETURN_UNEXPECTED_EXIT();
9492	}
9493
9494
9495	/**
9496	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9497	*/
9498	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9499	{
9500	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9501	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9502	AssertRCReturn(rc, rc);
9503
9504	PVM pVM = pVCpu->CTX_SUFF(pVM);
9505	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9506	if (RT_LIKELY(rc == VINF_SUCCESS))
9507	{
9508	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9509	Assert(pVmxTransient->cbInstr == 2);
9510	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9511	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9512	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9513	}
9514	else
9515	{
9516	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9517	rc = VERR_EM_INTERPRETER;
9518	}
9519	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9520	return rc;
9521	}
9522
9523
9524	/**
9525	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9526	*/
9527	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9528	{
9529	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9530	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9531	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9532	AssertRCReturn(rc, rc);
9533
9534	PVM pVM = pVCpu->CTX_SUFF(pVM);
9535	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9536	if (RT_LIKELY(rc == VINF_SUCCESS))
9537	{
9538	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9539	Assert(pVmxTransient->cbInstr == 3);
9540	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9541	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9542	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9543	}
9544	else
9545	{
9546	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9547	rc = VERR_EM_INTERPRETER;
9548	}
9549	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9550	return rc;
9551	}
9552
9553
9554	/**
9555	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9556	*/
9557	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9558	{
9559	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9560	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9561	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9562	AssertRCReturn(rc, rc);
9563
9564	PVM pVM = pVCpu->CTX_SUFF(pVM);
9565	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9566	if (RT_LIKELY(rc == VINF_SUCCESS))
9567	{
9568	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9569	Assert(pVmxTransient->cbInstr == 2);
9570	}
9571	else
9572	{
9573	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9574	rc = VERR_EM_INTERPRETER;
9575	}
9576	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9577	return rc;
9578	}
9579
9580
9581	/**
9582	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9583	*/
9584	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9585	{
9586	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9587	PVM pVM = pVCpu->CTX_SUFF(pVM);
9588	Assert(!pVM->hm.s.fNestedPaging);
9589
9590	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9591	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9592	AssertRCReturn(rc, rc);
9593
9594	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9595	rc = VBOXSTRICTRC_VAL(rc2);
9596	if (RT_LIKELY(rc == VINF_SUCCESS))
9597	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9598	else
9599	{
9600	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9601	pVmxTransient->uExitQualification, rc));
9602	}
9603	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9604	return rc;
9605	}
9606
9607
9608	/**
9609	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9610	*/
9611	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9612	{
9613	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9614	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9615	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9616	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9617	AssertRCReturn(rc, rc);
9618
9619	PVM pVM = pVCpu->CTX_SUFF(pVM);
9620	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9621	if (RT_LIKELY(rc == VINF_SUCCESS))
9622	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9623	else
9624	{
9625	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9626	rc = VERR_EM_INTERPRETER;
9627	}
9628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9629	return rc;
9630	}
9631
9632
9633	/**
9634	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9635	*/
9636	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9637	{
9638	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9639	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9640	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9641	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9642	AssertRCReturn(rc, rc);
9643
9644	PVM pVM = pVCpu->CTX_SUFF(pVM);
9645	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9646	rc = VBOXSTRICTRC_VAL(rc2);
9647	if (RT_LIKELY( rc == VINF_SUCCESS
9648	\|\| rc == VINF_EM_HALT))
9649	{
9650	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9651	AssertRCReturn(rc3, rc3);
9652
9653	if ( rc == VINF_EM_HALT
9654	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9655	{
9656	rc = VINF_SUCCESS;
9657	}
9658	}
9659	else
9660	{
9661	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9662	rc = VERR_EM_INTERPRETER;
9663	}
9664	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9665	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9667	return rc;
9668	}
9669
9670
9671	/**
9672	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9673	*/
9674	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9675	{
9676	/*
9677	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9678	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9679	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9680	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9681	*/
9682	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9683	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9684	HMVMX_RETURN_UNEXPECTED_EXIT();
9685	}
9686
9687
9688	/**
9689	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9690	*/
9691	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9692	{
9693	/*
9694	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9695	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9696	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9697	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9698	*/
9699	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9700	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9701	HMVMX_RETURN_UNEXPECTED_EXIT();
9702	}
9703
9704
9705	/**
9706	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9707	*/
9708	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9709	{
9710	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9711	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9712	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9713	HMVMX_RETURN_UNEXPECTED_EXIT();
9714	}
9715
9716
9717	/**
9718	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9719	*/
9720	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9721	{
9722	/*
9723	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9724	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9725	* See Intel spec. 25.3 "Other Causes of VM-exits".
9726	*/
9727	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9728	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9729	HMVMX_RETURN_UNEXPECTED_EXIT();
9730	}
9731
9732
9733	/**
9734	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9735	* VM-exit.
9736	*/
9737	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9738	{
9739	/*
9740	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9741	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9742	*
9743	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9744	* See Intel spec. "23.8 Restrictions on VMX operation".
9745	*/
9746	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9747	return VINF_SUCCESS;
9748	}
9749
9750
9751	/**
9752	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9753	* VM-exit.
9754	*/
9755	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9756	{
9757	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9758	return VINF_EM_RESET;
9759	}
9760
9761
9762	/**
9763	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9764	*/
9765	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9766	{
9767	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9768	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9769	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9770	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9771	AssertRCReturn(rc, rc);
9772
9773	pMixedCtx->rip++;
9774	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9775	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9776	rc = VINF_SUCCESS;
9777	else
9778	rc = VINF_EM_HALT;
9779
9780	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9781	return rc;
9782	}
9783
9784
9785	/**
9786	* VM-exit handler for instructions that result in a #UD exception delivered to
9787	* the guest.
9788	*/
9789	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9790	{
9791	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9792	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9793	return VINF_SUCCESS;
9794	}
9795
9796
9797	/**
9798	* VM-exit handler for expiry of the VMX preemption timer.
9799	*/
9800	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9801	{
9802	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9803
9804	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9805	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9806
9807	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9808	PVM pVM = pVCpu->CTX_SUFF(pVM);
9809	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9811	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9812	}
9813
9814
9815	/**
9816	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9817	*/
9818	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9819	{
9820	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9821
9822	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9823	/** @todo check if XSETBV is supported by the recompiler. */
9824	return VERR_EM_INTERPRETER;
9825	}
9826
9827
9828	/**
9829	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9830	*/
9831	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9832	{
9833	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9834
9835	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9836	/** @todo implement EMInterpretInvpcid() */
9837	return VERR_EM_INTERPRETER;
9838	}
9839
9840
9841	/**
9842	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9843	* Error VM-exit.
9844	*/
9845	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9846	{
9847	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9848	AssertRCReturn(rc, rc);
9849
9850	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9851	NOREF(uInvalidReason);
9852
9853	#ifdef VBOX_STRICT
9854	uint32_t uIntrState;
9855	HMVMXHCUINTREG uHCReg;
9856	uint64_t u64Val;
9857	uint32_t u32Val;
9858
9859	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9860	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9861	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9862	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9863	AssertRCReturn(rc, rc);
9864
9865	Log4(("uInvalidReason %u\n", uInvalidReason));
9866	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9867	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9868	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9869	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9870
9871	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9872	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9873	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9874	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9875	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9876	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9877	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9878	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9879	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9880	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9881	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9882	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9883	#else
9884	NOREF(pVmxTransient);
9885	#endif
9886
9887	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9888	return VERR_VMX_INVALID_GUEST_STATE;
9889	}
9890
9891
9892	/**
9893	* VM-exit handler for VM-entry failure due to an MSR-load
9894	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9895	*/
9896	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9897	{
9898	NOREF(pVmxTransient);
9899	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9900	HMVMX_RETURN_UNEXPECTED_EXIT();
9901	}
9902
9903
9904	/**
9905	* VM-exit handler for VM-entry failure due to a machine-check event
9906	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9907	*/
9908	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9909	{
9910	NOREF(pVmxTransient);
9911	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9912	HMVMX_RETURN_UNEXPECTED_EXIT();
9913	}
9914
9915
9916	/**
9917	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9918	* theory.
9919	*/
9920	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9921	{
9922	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9923	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9924	return VERR_VMX_UNDEFINED_EXIT_CODE;
9925	}
9926
9927
9928	/**
9929	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9930	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9931	* Conditional VM-exit.
9932	*/
9933	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9934	{
9935	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9936
9937	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9938	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9939	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9940	return VERR_EM_INTERPRETER;
9941	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9942	HMVMX_RETURN_UNEXPECTED_EXIT();
9943	}
9944
9945
9946	/**
9947	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9948	*/
9949	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9950	{
9951	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9952
9953	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9954	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9955	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9956	return VERR_EM_INTERPRETER;
9957	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9958	HMVMX_RETURN_UNEXPECTED_EXIT();
9959	}
9960
9961
9962	/**
9963	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9964	*/
9965	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9966	{
9967	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9968
9969	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9970	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9971	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9972	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9973	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9974	{
9975	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9976	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9977	}
9978	AssertRCReturn(rc, rc);
9979	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9980
9981	#ifdef VBOX_STRICT
9982	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9983	{
9984	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9985	{
9986	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9987	HMVMX_RETURN_UNEXPECTED_EXIT();
9988	}
9989	# if HC_ARCH_BITS == 64
9990	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
9991	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
9992	{
9993	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
9994	HMVMX_RETURN_UNEXPECTED_EXIT();
9995	}
9996	# endif
9997	}
9998	#endif
9999
10000	PVM pVM = pVCpu->CTX_SUFF(pVM);
10001	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10002	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
10003	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10004	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10005
10006	if (RT_LIKELY(rc == VINF_SUCCESS))
10007	{
10008	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10009	Assert(pVmxTransient->cbInstr == 2);
10010	}
10011	return rc;
10012	}
10013
10014
10015	/**
10016	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10017	*/
10018	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10019	{
10020	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10021	PVM pVM = pVCpu->CTX_SUFF(pVM);
10022	int rc = VINF_SUCCESS;
10023
10024	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10025	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10026	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10027	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10028	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10029	{
10030	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10031	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10032	}
10033	AssertRCReturn(rc, rc);
10034	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10035
10036	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10037	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10038	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10039
10040	if (RT_LIKELY(rc == VINF_SUCCESS))
10041	{
10042	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10043
10044	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10045	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10046	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10047	{
10048	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10049	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10050	EMInterpretWrmsr() changes it. */
10051	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10052	}
10053	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10054	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10055
10056	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10057	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10058	{
10059	switch (pMixedCtx->ecx)
10060	{
10061	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10062	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10063	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10064	case MSR_K8_FS_BASE: /* no break */
10065	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10066	default:
10067	{
10068	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10069	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10070	#if HC_ARCH_BITS == 64
10071	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10072	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10073	#endif
10074	break;
10075	}
10076	}
10077	}
10078	#ifdef VBOX_STRICT
10079	else
10080	{
10081	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10082	switch (pMixedCtx->ecx)
10083	{
10084	case MSR_IA32_SYSENTER_CS:
10085	case MSR_IA32_SYSENTER_EIP:
10086	case MSR_IA32_SYSENTER_ESP:
10087	case MSR_K8_FS_BASE:
10088	case MSR_K8_GS_BASE:
10089	{
10090	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10091	HMVMX_RETURN_UNEXPECTED_EXIT();
10092	}
10093
10094	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10095	default:
10096	{
10097	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10098	{
10099	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10100	pMixedCtx->ecx));
10101	HMVMX_RETURN_UNEXPECTED_EXIT();
10102	}
10103
10104	#if HC_ARCH_BITS == 64
10105	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10106	{
10107	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10108	HMVMX_RETURN_UNEXPECTED_EXIT();
10109	}
10110	#endif
10111	break;
10112	}
10113	}
10114	}
10115	#endif /* VBOX_STRICT */
10116	}
10117	return rc;
10118	}
10119
10120
10121	/**
10122	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10123	*/
10124	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10125	{
10126	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10127
10128	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10130	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10131	return VERR_EM_INTERPRETER;
10132	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10133	HMVMX_RETURN_UNEXPECTED_EXIT();
10134	}
10135
10136
10137	/**
10138	* VM-exit handler for when the TPR value is lowered below the specified
10139	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10140	*/
10141	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10142	{
10143	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10144	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10145
10146	/*
10147	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10148	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10149	* resume guest execution.
10150	*/
10151	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10152	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10153	return VINF_SUCCESS;
10154	}
10155
10156
10157	/**
10158	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10159	* VM-exit.
10160	*
10161	* @retval VINF_SUCCESS when guest execution can continue.
10162	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10163	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10164	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10165	* recompiler.
10166	*/
10167	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10168	{
10169	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10170	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10171	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10172	AssertRCReturn(rc, rc);
10173
10174	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10175	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10176	PVM pVM = pVCpu->CTX_SUFF(pVM);
10177	switch (uAccessType)
10178	{
10179	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10180	{
10181	#if 0
10182	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10183	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10184	#else
10185	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10186	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10187	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10188	#endif
10189	AssertRCReturn(rc, rc);
10190
10191	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10192	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10193	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10194	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10195
10196	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10197	{
10198	case 0: /* CR0 */
10199	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10200	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10201	break;
10202	case 2: /* CR2 */
10203	/* Nothing to do here, CR2 it's not part of the VMCS. */
10204	break;
10205	case 3: /* CR3 */
10206	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10207	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10208	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10209	break;
10210	case 4: /* CR4 */
10211	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10212	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10213	break;
10214	case 8: /* CR8 */
10215	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10216	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10217	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10218	break;
10219	default:
10220	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10221	break;
10222	}
10223
10224	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10225	break;
10226	}
10227
10228	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10229	{
10230	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10231	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10232	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10233	AssertRCReturn(rc, rc);
10234	Assert( !pVM->hm.s.fNestedPaging
10235	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10236	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10237
10238	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10239	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10240	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10241
10242	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10243	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10244	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10245	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10246	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10247	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10248	break;
10249	}
10250
10251	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10252	{
10253	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10254	AssertRCReturn(rc, rc);
10255	rc = EMInterpretCLTS(pVM, pVCpu);
10256	AssertRCReturn(rc, rc);
10257	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10258	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10259	Log4(("CRX CLTS write rc=%d\n", rc));
10260	break;
10261	}
10262
10263	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10264	{
10265	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10266	AssertRCReturn(rc, rc);
10267	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10268	if (RT_LIKELY(rc == VINF_SUCCESS))
10269	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10270	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10271	Log4(("CRX LMSW write rc=%d\n", rc));
10272	break;
10273	}
10274
10275	default:
10276	{
10277	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10278	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10279	}
10280	}
10281
10282	/* Validate possible error codes. */
10283	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10284	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10285	if (RT_SUCCESS(rc))
10286	{
10287	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10288	AssertRCReturn(rc2, rc2);
10289	}
10290
10291	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10292	return rc;
10293	}
10294
10295
10296	/**
10297	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10298	* VM-exit.
10299	*/
10300	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10301	{
10302	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10303	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10304
10305	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10306	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10307	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10308	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10309	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10310	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10311	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10312	AssertRCReturn(rc2, rc2);
10313
10314	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10315	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10316	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10317	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10318	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10319	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10320	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10321
10322	/* I/O operation lookup arrays. */
10323	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10324	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10325
10326	VBOXSTRICTRC rcStrict;
10327	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10328	const uint32_t cbInstr = pVmxTransient->cbInstr;
10329	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10330	PVM pVM = pVCpu->CTX_SUFF(pVM);
10331	if (fIOString)
10332	{
10333	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10334	/*
10335	* INS/OUTS - I/O String instruction.
10336	*
10337	* Use instruction-information if available, otherwise fall back on
10338	* interpreting the instruction.
10339	*/
10340	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10341	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10342	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10343	{
10344	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10345	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10346	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10347	AssertRCReturn(rc2, rc2);
10348	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10349	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10350	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10351	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10352	if (fIOWrite)
10353	{
10354	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10355	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10356	}
10357	else
10358	{
10359	/*
10360	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10361	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10362	* See Intel Instruction spec. for "INS".
10363	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10364	*/
10365	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10366	}
10367	}
10368	else
10369	{
10370	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10371	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10372	AssertRCReturn(rc2, rc2);
10373	rcStrict = IEMExecOne(pVCpu);
10374	}
10375	/** @todo IEM needs to be setting these flags somehow. */
10376	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10377	fUpdateRipAlready = true;
10378	#else
10379	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10380	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10381	if (RT_SUCCESS(rcStrict))
10382	{
10383	if (fIOWrite)
10384	{
10385	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10386	(DISCPUMODE)pDis->uAddrMode, cbValue);
10387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10388	}
10389	else
10390	{
10391	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10392	(DISCPUMODE)pDis->uAddrMode, cbValue);
10393	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10394	}
10395	}
10396	else
10397	{
10398	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10399	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10400	}
10401	#endif
10402	}
10403	else
10404	{
10405	/*
10406	* IN/OUT - I/O instruction.
10407	*/
10408	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10409	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10410	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10411	if (fIOWrite)
10412	{
10413	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10414	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10415	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10417	}
10418	else
10419	{
10420	uint32_t u32Result = 0;
10421	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10422	if (IOM_SUCCESS(rcStrict))
10423	{
10424	/* Save result of I/O IN instr. in AL/AX/EAX. */
10425	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10426	}
10427	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10428	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10429	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10430	}
10431	}
10432
10433	if (IOM_SUCCESS(rcStrict))
10434	{
10435	if (!fUpdateRipAlready)
10436	{
10437	pMixedCtx->rip += cbInstr;
10438	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10439	}
10440
10441	/*
10442	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10443	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10444	*/
10445	if (fIOString)
10446	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10447
10448	/*
10449	* If any I/O breakpoints are armed, we need to check if one triggered
10450	* and take appropriate action.
10451	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10452	*/
10453	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10454	AssertRCReturn(rc2, rc2);
10455
10456	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10457	* execution engines about whether hyper BPs and such are pending. */
10458	uint32_t const uDr7 = pMixedCtx->dr[7];
10459	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10460	&& X86_DR7_ANY_RW_IO(uDr7)
10461	&& (pMixedCtx->cr4 & X86_CR4_DE))
10462	\|\| DBGFBpIsHwIoArmed(pVM)))
10463	{
10464	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10465
10466	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10467	VMMRZCallRing3Disable(pVCpu);
10468	HM_DISABLE_PREEMPT_IF_NEEDED();
10469
10470	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10471
10472	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10473	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10474	{
10475	/* Raise #DB. */
10476	if (fIsGuestDbgActive)
10477	ASMSetDR6(pMixedCtx->dr[6]);
10478	if (pMixedCtx->dr[7] != uDr7)
10479	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10480
10481	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10482	}
10483	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10484	else if ( rcStrict2 != VINF_SUCCESS
10485	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10486	rcStrict = rcStrict2;
10487
10488	HM_RESTORE_PREEMPT_IF_NEEDED();
10489	VMMRZCallRing3Enable(pVCpu);
10490	}
10491	}
10492
10493	#ifdef DEBUG
10494	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10495	Assert(!fIOWrite);
10496	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10497	Assert(fIOWrite);
10498	else
10499	{
10500	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10501	* statuses, that the VMM device and some others may return. See
10502	* IOM_SUCCESS() for guidance. */
10503	AssertMsg( RT_FAILURE(rcStrict)
10504	\|\| rcStrict == VINF_SUCCESS
10505	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10506	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10507	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10508	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10509	}
10510	#endif
10511
10512	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10513	return VBOXSTRICTRC_TODO(rcStrict);
10514	}
10515
10516
10517	/**
10518	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10519	* VM-exit.
10520	*/
10521	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10522	{
10523	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10524
10525	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10526	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10527	AssertRCReturn(rc, rc);
10528	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10529	{
10530	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10531	AssertRCReturn(rc, rc);
10532	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10533	{
10534	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10535
10536	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10537	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10538	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10539	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10540	{
10541	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10542	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10543
10544	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10545	Assert(!pVCpu->hm.s.Event.fPending);
10546	pVCpu->hm.s.Event.fPending = true;
10547	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10548	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10549	AssertRCReturn(rc, rc);
10550	if (fErrorCodeValid)
10551	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10552	else
10553	pVCpu->hm.s.Event.u32ErrCode = 0;
10554	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10555	&& uVector == X86_XCPT_PF)
10556	{
10557	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10558	}
10559
10560	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10561	}
10562	}
10563	}
10564
10565	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10566	* emulation. */
10567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10568	return VERR_EM_INTERPRETER;
10569	}
10570
10571
10572	/**
10573	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10574	*/
10575	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10576	{
10577	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10578	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10579	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10580	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10581	AssertRCReturn(rc, rc);
10582	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10583	return VINF_EM_DBG_STEPPED;
10584	}
10585
10586
10587	/**
10588	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10589	*/
10590	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10591	{
10592	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10593
10594	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10595	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10596	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10597	return VINF_SUCCESS;
10598	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10599	return rc;
10600
10601	#if 0
10602	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10603	* just sync the whole thing. */
10604	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10605	#else
10606	/* Aggressive state sync. for now. */
10607	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10608	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10609	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10610	#endif
10611	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10612	AssertRCReturn(rc, rc);
10613
10614	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10615	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10616	switch (uAccessType)
10617	{
10618	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10619	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10620	{
10621	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10622	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10623	{
10624	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10625	}
10626
10627	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10628	GCPhys &= PAGE_BASE_GC_MASK;
10629	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10630	PVM pVM = pVCpu->CTX_SUFF(pVM);
10631	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10632	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10633
10634	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10635	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10636	CPUMCTX2CORE(pMixedCtx), GCPhys);
10637	rc = VBOXSTRICTRC_VAL(rc2);
10638	Log4(("ApicAccess rc=%d\n", rc));
10639	if ( rc == VINF_SUCCESS
10640	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10641	\|\| rc == VERR_PAGE_NOT_PRESENT)
10642	{
10643	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10644	\| HM_CHANGED_GUEST_RSP
10645	\| HM_CHANGED_GUEST_RFLAGS
10646	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10647	rc = VINF_SUCCESS;
10648	}
10649	break;
10650	}
10651
10652	default:
10653	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10654	rc = VINF_EM_RAW_EMULATE_INSTR;
10655	break;
10656	}
10657
10658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10659	return rc;
10660	}
10661
10662
10663	/**
10664	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10665	* VM-exit.
10666	*/
10667	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10668	{
10669	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10670
10671	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10672	if (pVmxTransient->fWasGuestDebugStateActive)
10673	{
10674	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10675	HMVMX_RETURN_UNEXPECTED_EXIT();
10676	}
10677
10678	int rc = VERR_INTERNAL_ERROR_5;
10679	if ( !DBGFIsStepping(pVCpu)
10680	&& !pVCpu->hm.s.fSingleInstruction
10681	&& !pVmxTransient->fWasHyperDebugStateActive)
10682	{
10683	/* Don't intercept MOV DRx and #DB any more. */
10684	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10685	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10686	AssertRCReturn(rc, rc);
10687
10688	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10689	{
10690	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10691	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10692	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10693	AssertRCReturn(rc, rc);
10694	#endif
10695	}
10696
10697	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10698	VMMRZCallRing3Disable(pVCpu);
10699	HM_DISABLE_PREEMPT_IF_NEEDED();
10700
10701	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10702	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10703	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10704
10705	HM_RESTORE_PREEMPT_IF_NEEDED();
10706	VMMRZCallRing3Enable(pVCpu);
10707
10708	#ifdef VBOX_WITH_STATISTICS
10709	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10710	AssertRCReturn(rc, rc);
10711	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10712	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10713	else
10714	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10715	#endif
10716	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10717	return VINF_SUCCESS;
10718	}
10719
10720	/*
10721	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10722	* Update the segment registers and DR7 from the CPU.
10723	*/
10724	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10725	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10726	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10727	AssertRCReturn(rc, rc);
10728	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10729
10730	PVM pVM = pVCpu->CTX_SUFF(pVM);
10731	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10732	{
10733	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10734	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10735	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10736	if (RT_SUCCESS(rc))
10737	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10738	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10739	}
10740	else
10741	{
10742	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10743	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10744	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10745	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10746	}
10747
10748	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10749	if (RT_SUCCESS(rc))
10750	{
10751	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10752	AssertRCReturn(rc2, rc2);
10753	}
10754	return rc;
10755	}
10756
10757
10758	/**
10759	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10760	* Conditional VM-exit.
10761	*/
10762	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10763	{
10764	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10765	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10766
10767	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10768	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10769	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10770	return VINF_SUCCESS;
10771	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10772	return rc;
10773
10774	RTGCPHYS GCPhys = 0;
10775	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10776
10777	#if 0
10778	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10779	#else
10780	/* Aggressive state sync. for now. */
10781	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10782	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10783	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10784	#endif
10785	AssertRCReturn(rc, rc);
10786
10787	/*
10788	* If we succeed, resume guest execution.
10789	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10790	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10791	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10792	* weird case. See @bugref{6043}.
10793	*/
10794	PVM pVM = pVCpu->CTX_SUFF(pVM);
10795	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10796	rc = VBOXSTRICTRC_VAL(rc2);
10797	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10798	if ( rc == VINF_SUCCESS
10799	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10800	\|\| rc == VERR_PAGE_NOT_PRESENT)
10801	{
10802	/* Successfully handled MMIO operation. */
10803	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10804	\| HM_CHANGED_GUEST_RSP
10805	\| HM_CHANGED_GUEST_RFLAGS
10806	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10807	rc = VINF_SUCCESS;
10808	}
10809	return rc;
10810	}
10811
10812
10813	/**
10814	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10815	* VM-exit.
10816	*/
10817	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10818	{
10819	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10820	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10821
10822	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10823	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10824	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10825	return VINF_SUCCESS;
10826	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10827	return rc;
10828
10829	RTGCPHYS GCPhys = 0;
10830	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10831	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10832	#if 0
10833	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10834	#else
10835	/* Aggressive state sync. for now. */
10836	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10837	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10838	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10839	#endif
10840	AssertRCReturn(rc, rc);
10841
10842	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10843	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10844
10845	RTGCUINT uErrorCode = 0;
10846	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10847	uErrorCode \|= X86_TRAP_PF_ID;
10848	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10849	uErrorCode \|= X86_TRAP_PF_RW;
10850	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10851	uErrorCode \|= X86_TRAP_PF_P;
10852
10853	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10854
10855	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10856	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10857
10858	/* Handle the pagefault trap for the nested shadow table. */
10859	PVM pVM = pVCpu->CTX_SUFF(pVM);
10860	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10861	TRPMResetTrap(pVCpu);
10862
10863	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10864	if ( rc == VINF_SUCCESS
10865	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10866	\|\| rc == VERR_PAGE_NOT_PRESENT)
10867	{
10868	/* Successfully synced our nested page tables. */
10869	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10870	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10871	\| HM_CHANGED_GUEST_RSP
10872	\| HM_CHANGED_GUEST_RFLAGS);
10873	return VINF_SUCCESS;
10874	}
10875
10876	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10877	return rc;
10878	}
10879
10880	/** @} */
10881
10882	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10883	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10884	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10885
10886	/** @name VM-exit exception handlers.
10887	* @{
10888	*/
10889
10890	/**
10891	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10892	*/
10893	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10894	{
10895	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10896	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10897
10898	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10899	AssertRCReturn(rc, rc);
10900
10901	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10902	{
10903	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10904	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10905
10906	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
10907	* provides VM-exit instruction length. If this causes problem later,
10908	* disassemble the instruction like it's done on AMD-V. */
10909	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10910	AssertRCReturn(rc2, rc2);
10911	return rc;
10912	}
10913
10914	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10915	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10916	return rc;
10917	}
10918
10919
10920	/**
10921	* VM-exit exception handler for #BP (Breakpoint exception).
10922	*/
10923	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10924	{
10925	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10926	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10927
10928	/** @todo Try optimize this by not saving the entire guest state unless
10929	* really needed. */
10930	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10931	AssertRCReturn(rc, rc);
10932
10933	PVM pVM = pVCpu->CTX_SUFF(pVM);
10934	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10935	if (rc == VINF_EM_RAW_GUEST_TRAP)
10936	{
10937	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10938	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10939	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10940	AssertRCReturn(rc, rc);
10941
10942	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10943	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10944	}
10945
10946	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10947	return rc;
10948	}
10949
10950
10951	/**
10952	* VM-exit exception handler for #DB (Debug exception).
10953	*/
10954	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10955	{
10956	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10958	Log6(("XcptDB\n"));
10959
10960	/*
10961	* Get the DR6-like values from the exit qualification and pass it to DBGF
10962	* for processing.
10963	*/
10964	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10965	AssertRCReturn(rc, rc);
10966
10967	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10968	uint64_t uDR6 = X86_DR6_INIT_VAL;
10969	uDR6 \|= ( pVmxTransient->uExitQualification
10970	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10971
10972	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10973	if (rc == VINF_EM_RAW_GUEST_TRAP)
10974	{
10975	/*
10976	* The exception was for the guest. Update DR6, DR7.GD and
10977	* IA32_DEBUGCTL.LBR before forwarding it.
10978	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10979	*/
10980	VMMRZCallRing3Disable(pVCpu);
10981	HM_DISABLE_PREEMPT_IF_NEEDED();
10982
10983	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10984	pMixedCtx->dr[6] \|= uDR6;
10985	if (CPUMIsGuestDebugStateActive(pVCpu))
10986	ASMSetDR6(pMixedCtx->dr[6]);
10987
10988	HM_RESTORE_PREEMPT_IF_NEEDED();
10989	VMMRZCallRing3Enable(pVCpu);
10990
10991	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10992	AssertRCReturn(rc, rc);
10993
10994	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10995	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10996
10997	/* Paranoia. */
10998	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10999	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
11000
11001	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11002	AssertRCReturn(rc, rc);
11003
11004	/*
11005	* Raise #DB in the guest.
11006	*
11007	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11008	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11009	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11010	*
11011	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11012	*/
11013	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11014	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11015	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11016	AssertRCReturn(rc, rc);
11017	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11018	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11019	return VINF_SUCCESS;
11020	}
11021
11022	/*
11023	* Not a guest trap, must be a hypervisor related debug event then.
11024	* Update DR6 in case someone is interested in it.
11025	*/
11026	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11027	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11028	CPUMSetHyperDR6(pVCpu, uDR6);
11029
11030	return rc;
11031	}
11032
11033
11034	/**
11035	* VM-exit exception handler for #NM (Device-not-available exception: floating
11036	* point exception).
11037	*/
11038	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11039	{
11040	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11041
11042	/* We require CR0 and EFER. EFER is always up-to-date. */
11043	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11044	AssertRCReturn(rc, rc);
11045
11046	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11047	VMMRZCallRing3Disable(pVCpu);
11048	HM_DISABLE_PREEMPT_IF_NEEDED();
11049
11050	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11051	if (pVmxTransient->fWasGuestFPUStateActive)
11052	{
11053	rc = VINF_EM_RAW_GUEST_TRAP;
11054	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11055	}
11056	else
11057	{
11058	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11059	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11060	#endif
11061	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11062	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11063	}
11064
11065	HM_RESTORE_PREEMPT_IF_NEEDED();
11066	VMMRZCallRing3Enable(pVCpu);
11067
11068	if (rc == VINF_SUCCESS)
11069	{
11070	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11071	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11072	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11073	pVCpu->hm.s.fUseGuestFpu = true;
11074	}
11075	else
11076	{
11077	/* Forward #NM to the guest. */
11078	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11079	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11080	AssertRCReturn(rc, rc);
11081	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11082	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11084	}
11085
11086	return VINF_SUCCESS;
11087	}
11088
11089
11090	/**
11091	* VM-exit exception handler for #GP (General-protection exception).
11092	*
11093	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11094	*/
11095	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11096	{
11097	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11098	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11099
11100	int rc = VERR_INTERNAL_ERROR_5;
11101	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11102	{
11103	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11104	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11105	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11106	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11107	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11108	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11109	AssertRCReturn(rc, rc);
11110	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11111	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11112	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11113	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11114	return rc;
11115	#else
11116	/* We don't intercept #GP. */
11117	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11118	NOREF(pVmxTransient);
11119	return VERR_VMX_UNEXPECTED_EXCEPTION;
11120	#endif
11121	}
11122
11123	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11124	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11125
11126	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11127	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11128	AssertRCReturn(rc, rc);
11129
11130	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11131	uint32_t cbOp = 0;
11132	PVM pVM = pVCpu->CTX_SUFF(pVM);
11133	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11134	if (RT_SUCCESS(rc))
11135	{
11136	rc = VINF_SUCCESS;
11137	Assert(cbOp == pDis->cbInstr);
11138	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11139	switch (pDis->pCurInstr->uOpcode)
11140	{
11141	case OP_CLI:
11142	{
11143	pMixedCtx->eflags.Bits.u1IF = 0;
11144	pMixedCtx->rip += pDis->cbInstr;
11145	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11146	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11147	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11148	break;
11149	}
11150
11151	case OP_STI:
11152	{
11153	pMixedCtx->eflags.Bits.u1IF = 1;
11154	pMixedCtx->rip += pDis->cbInstr;
11155	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11156	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11157	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11158	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11159	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11160	break;
11161	}
11162
11163	case OP_HLT:
11164	{
11165	rc = VINF_EM_HALT;
11166	pMixedCtx->rip += pDis->cbInstr;
11167	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11168	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11169	break;
11170	}
11171
11172	case OP_POPF:
11173	{
11174	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11175	uint32_t cbParm = 0;
11176	uint32_t uMask = 0;
11177	bool fAlreadyStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11178	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11179	{
11180	cbParm = 4;
11181	uMask = 0xffffffff;
11182	}
11183	else
11184	{
11185	cbParm = 2;
11186	uMask = 0xffff;
11187	}
11188
11189	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11190	RTGCPTR GCPtrStack = 0;
11191	X86EFLAGS Eflags;
11192	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11193	&GCPtrStack);
11194	if (RT_SUCCESS(rc))
11195	{
11196	Assert(sizeof(Eflags.u32) >= cbParm);
11197	Eflags.u32 = 0;
11198	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11199	}
11200	if (RT_FAILURE(rc))
11201	{
11202	rc = VERR_EM_INTERPRETER;
11203	break;
11204	}
11205	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11206	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11207	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11208	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11209	pMixedCtx->esp += cbParm;
11210	pMixedCtx->esp &= uMask;
11211	pMixedCtx->rip += pDis->cbInstr;
11212	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11213	\| HM_CHANGED_GUEST_RSP
11214	\| HM_CHANGED_GUEST_RFLAGS);
11215
11216	/* Only generate a debug execption after POPF if the guest is already stepping over POPF and
11217	POPF restores EFLAGS.TF. The CPU looks at the EFLAGS.TF after the instruction is done manipulating it. */
11218	if ( fAlreadyStepping
11219	&& pMixedCtx->eflags.Bits.u1TF)
11220	{
11221	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11222	}
11223
11224	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11225	break;
11226	}
11227
11228	case OP_PUSHF:
11229	{
11230	uint32_t cbParm = 0;
11231	uint32_t uMask = 0;
11232	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11233	{
11234	cbParm = 4;
11235	uMask = 0xffffffff;
11236	}
11237	else
11238	{
11239	cbParm = 2;
11240	uMask = 0xffff;
11241	}
11242
11243	/* Get the stack pointer & push the contents of eflags onto the stack. */
11244	RTGCPTR GCPtrStack = 0;
11245	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11246	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11247	if (RT_FAILURE(rc))
11248	{
11249	rc = VERR_EM_INTERPRETER;
11250	break;
11251	}
11252	X86EFLAGS Eflags = pMixedCtx->eflags;
11253	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11254	Eflags.Bits.u1RF = 0;
11255	Eflags.Bits.u1VM = 0;
11256
11257	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11258	if (RT_FAILURE(rc))
11259	{
11260	rc = VERR_EM_INTERPRETER;
11261	break;
11262	}
11263	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11264	pMixedCtx->esp -= cbParm;
11265	pMixedCtx->esp &= uMask;
11266	pMixedCtx->rip += pDis->cbInstr;
11267	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11268	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11269	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11270	break;
11271	}
11272
11273	case OP_IRET:
11274	{
11275	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11276	* instruction reference. */
11277	RTGCPTR GCPtrStack = 0;
11278	uint32_t uMask = 0xffff;
11279	uint16_t aIretFrame[3];
11280	bool fAlreadyStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11281	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11282	{
11283	rc = VERR_EM_INTERPRETER;
11284	break;
11285	}
11286	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11287	&GCPtrStack);
11288	if (RT_SUCCESS(rc))
11289	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11290	if (RT_FAILURE(rc))
11291	{
11292	rc = VERR_EM_INTERPRETER;
11293	break;
11294	}
11295	pMixedCtx->eip = 0;
11296	pMixedCtx->ip = aIretFrame[0];
11297	pMixedCtx->cs.Sel = aIretFrame[1];
11298	pMixedCtx->cs.ValidSel = aIretFrame[1];
11299	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11300	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11301	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11302	pMixedCtx->sp += sizeof(aIretFrame);
11303	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11304	\| HM_CHANGED_GUEST_SEGMENT_REGS
11305	\| HM_CHANGED_GUEST_RSP
11306	\| HM_CHANGED_GUEST_RFLAGS);
11307
11308	/* Only generate a debug execption after IRET if the guest is already stepping over IRET and
11309	IRET restores EFLAGS.TF. The CPU looks at the EFLAGS.TF after the instruction is done manipulating it. */
11310	if ( fAlreadyStepping
11311	&& pMixedCtx->eflags.Bits.u1TF)
11312	{
11313	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11314	}
11315
11316	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11318	break;
11319	}
11320
11321	case OP_INT:
11322	{
11323	uint16_t uVector = pDis->Param1.uValue & 0xff;
11324	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11325	/* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11326	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11327	break;
11328	}
11329
11330	case OP_INTO:
11331	{
11332	if (pMixedCtx->eflags.Bits.u1OF)
11333	{
11334	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11335	/* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11337	}
11338	break;
11339	}
11340
11341	default:
11342	{
11343	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11344	EMCODETYPE_SUPERVISOR);
11345	rc = VBOXSTRICTRC_VAL(rc2);
11346	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11347	/** @todo We have to set pending-debug exceptions here when the guest is
11348	* single-stepping depending on the instruction that was interpreted. */
11349	Log4(("#GP rc=%Rrc\n", rc));
11350	break;
11351	}
11352	}
11353	}
11354	else
11355	rc = VERR_EM_INTERPRETER;
11356
11357	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11358	("#GP Unexpected rc=%Rrc\n", rc));
11359	return rc;
11360	}
11361
11362
11363	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11364	/**
11365	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11366	* the exception reported in the VMX transient structure back into the VM.
11367	*
11368	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11369	* up-to-date.
11370	*/
11371	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11372	{
11373	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11374
11375	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11376	hmR0VmxCheckExitDueToEventDelivery(). */
11377	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11378	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11379	AssertRCReturn(rc, rc);
11380	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11381
11382	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11383	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11384	return VINF_SUCCESS;
11385	}
11386	#endif
11387
11388
11389	/**
11390	* VM-exit exception handler for #PF (Page-fault exception).
11391	*/
11392	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11393	{
11394	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11395	PVM pVM = pVCpu->CTX_SUFF(pVM);
11396	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11397	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11398	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11399	AssertRCReturn(rc, rc);
11400
11401	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11402	if (pVM->hm.s.fNestedPaging)
11403	{
11404	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11405	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11406	{
11407	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11408	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11409	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11410	}
11411	else
11412	{
11413	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11414	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11415	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11416	}
11417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11418	return rc;
11419	}
11420	#else
11421	Assert(!pVM->hm.s.fNestedPaging);
11422	NOREF(pVM);
11423	#endif
11424
11425	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11426	AssertRCReturn(rc, rc);
11427
11428	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11429	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11430
11431	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11432	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11433	(RTGCPTR)pVmxTransient->uExitQualification);
11434
11435	Log4(("#PF: rc=%Rrc\n", rc));
11436	if (rc == VINF_SUCCESS)
11437	{
11438	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11439	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11440	* memory? We don't update the whole state here... */
11441	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11442	\| HM_CHANGED_GUEST_RSP
11443	\| HM_CHANGED_GUEST_RFLAGS
11444	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11445	TRPMResetTrap(pVCpu);
11446	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11447	return rc;
11448	}
11449	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11450	{
11451	if (!pVmxTransient->fVectoringPF)
11452	{
11453	/* It's a guest page fault and needs to be reflected to the guest. */
11454	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11455	TRPMResetTrap(pVCpu);
11456	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11457	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11458	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11459	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11460	}
11461	else
11462	{
11463	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11464	TRPMResetTrap(pVCpu);
11465	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11466	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11467	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11468	}
11469
11470	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11471	return VINF_SUCCESS;
11472	}
11473
11474	TRPMResetTrap(pVCpu);
11475	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11476	return rc;
11477	}
11478
11479	/** @} */
11480

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 50275

Download in other formats: