HMVMXR0.cpp@ 49699

Last change on this file since 49699 was 49699, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Update guest TSC_AUX before auto-load/store.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 471.4 KB

Line
1	/* $Id: HMVMXR0.cpp 49699 2013-11-28 12:52:47Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	# define HMVMX_SAVE_FULL_GUEST_STATE
40	# define HMVMX_SYNC_FULL_GUEST_STATE
41	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	# define HMVMX_ALWAYS_TRAP_PF
44	# define HMVMX_ALWAYS_SWAP_FPU_STATE
45	# define HMVMX_ALWAYS_FLUSH_TLB
46	#endif
47
48
49	/*******************************************************************************
50	* Defined Constants And Macros *
51	*******************************************************************************/
52	#if defined(RT_ARCH_AMD64)
53	# define HMVMX_IS_64BIT_HOST_MODE() (true)
54	typedef RTHCUINTREG HMVMXHCUINTREG;
55	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56	extern "C" uint32_t g_fVMXIs64bitHost;
57	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58	typedef uint64_t HMVMXHCUINTREG;
59	#else
60	# define HMVMX_IS_64BIT_HOST_MODE() (false)
61	typedef RTHCUINTREG HMVMXHCUINTREG;
62	#endif
63
64	/** Use the function table. */
65	#define HMVMX_USE_FUNCTION_TABLE
66
67	/** Determine which tagged-TLB flush handler to use. */
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
70	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
71	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
72
73	/** @name Updated-guest-state flags.
74	* @{ */
75	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
76	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
77	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
78	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
79	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
80	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
81	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
82	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
83	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
84	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
85	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
86	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
91	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
109	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
110	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113	/** @} */
114
115	/** @name
116	* Flags to skip redundant reads of some common VMCS fields that are not part of
117	* the guest-CPU state but are in the transient structure.
118	*/
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
126	/** @} */
127
128	/** @name
129	* States of the VMCS.
130	*
131	* This does not reflect all possible VMCS states but currently only those
132	* needed for maintaining the VMCS consistently even when thread-context hooks
133	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
134	*/
135	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
136	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
137	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
138	/** @} */
139
140	/**
141	* Exception bitmap mask for real-mode guests (real-on-v86).
142	*
143	* We need to intercept all exceptions manually (except #PF). #NM is also
144	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
145	* even in real-mode if we have Nested Paging support.
146	*/
147	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
148	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
149	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
150	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
151	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
153	\| RT_BIT(X86_XCPT_XF))
154
155	/**
156	* Exception bitmap mask for all contributory exceptions.
157	*
158	* Page fault is deliberately excluded here as it's conditional as to whether
159	* it's contributory or benign. Page faults are handled separately.
160	*/
161	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
162	\| RT_BIT(X86_XCPT_DE))
163
164	/** Maximum VM-instruction error number. */
165	#define HMVMX_INSTR_ERROR_MAX 28
166
167	/** Profiling macro. */
168	#ifdef HM_PROFILE_EXIT_DISPATCH
169	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171	#else
172	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174	#endif
175
176	/** Assert that preemption is disabled or covered by thread-context hooks. */
177	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
178	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180	/** Assert that we haven't migrated CPUs when thread-context hooks are not
181	* used. */
182	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184	("Illegal migration! Entered on CPU %u Current %u\n", \
185	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187	/** Helper macro for VM-exit handlers called unexpectedly. */
188	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189	do { \
190	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191	return VERR_VMX_UNEXPECTED_EXIT; \
192	} while (0)
193
194
195	/*******************************************************************************
196	* Structures and Typedefs *
197	*******************************************************************************/
198	/**
199	* VMX transient state.
200	*
201	* A state structure for holding miscellaneous information across
202	* VMX non-root operation and restored after the transition.
203	*/
204	typedef struct VMXTRANSIENT
205	{
206	/** The host's rflags/eflags. */
207	RTCCUINTREG uEflags;
208	#if HC_ARCH_BITS == 32
209	uint32_t u32Alignment0;
210	#endif
211	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
212	uint64_t u64LStarMsr;
213	/** The guest's TPR value used for TPR shadowing. */
214	uint8_t u8GuestTpr;
215	/** Alignment. */
216	uint8_t abAlignment0[7];
217
218	/** The basic VM-exit reason. */
219	uint16_t uExitReason;
220	/** Alignment. */
221	uint16_t u16Alignment0;
222	/** The VM-exit interruption error code. */
223	uint32_t uExitIntErrorCode;
224	/** The VM-exit exit qualification. */
225	uint64_t uExitQualification;
226
227	/** The VM-exit interruption-information field. */
228	uint32_t uExitIntInfo;
229	/** The VM-exit instruction-length field. */
230	uint32_t cbInstr;
231	/** The VM-exit instruction-information field. */
232	union
233	{
234	/** Plain unsigned int representation. */
235	uint32_t u;
236	/** INS and OUTS information. */
237	struct
238	{
239	uint32_t u6Reserved0 : 7;
240	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241	uint32_t u3AddrSize : 3;
242	uint32_t u5Reserved1 : 5;
243	/** The segment register (X86_SREG_XXX). */
244	uint32_t iSegReg : 3;
245	uint32_t uReserved2 : 14;
246	} StrIo;
247	} ExitInstrInfo;
248	/** Whether the VM-entry failed or not. */
249	bool fVMEntryFailed;
250	/** Alignment. */
251	uint8_t abAlignment1[3];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest FPU was active at the time of VM-exit. */
269	bool fWasGuestFPUStateActive;
270	/** Whether the guest debug state was active at the time of VM-exit. */
271	bool fWasGuestDebugStateActive;
272	/** Whether the hyper debug state was active at the time of VM-exit. */
273	bool fWasHyperDebugStateActive;
274	/** Whether TSC-offsetting should be setup before VM-entry. */
275	bool fUpdateTscOffsettingAndPreemptTimer;
276	/** Whether the VM-exit was caused by a page-fault during delivery of a
277	* contributory exception or a page-fault. */
278	bool fVectoringPF;
279	} VMXTRANSIENT;
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
284	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
285	/** Pointer to VMX transient state. */
286	typedef VMXTRANSIENT *PVMXTRANSIENT;
287
288
289	/**
290	* MSR-bitmap read permissions.
291	*/
292	typedef enum VMXMSREXITREAD
293	{
294	/** Reading this MSR causes a VM-exit. */
295	VMXMSREXIT_INTERCEPT_READ = 0xb,
296	/** Reading this MSR does not cause a VM-exit. */
297	VMXMSREXIT_PASSTHRU_READ
298	} VMXMSREXITREAD;
299	/** Pointer to MSR-bitmap read permissions. */
300	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
301
302	/**
303	* MSR-bitmap write permissions.
304	*/
305	typedef enum VMXMSREXITWRITE
306	{
307	/** Writing to this MSR causes a VM-exit. */
308	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
309	/** Writing to this MSR does not cause a VM-exit. */
310	VMXMSREXIT_PASSTHRU_WRITE
311	} VMXMSREXITWRITE;
312	/** Pointer to MSR-bitmap write permissions. */
313	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
314
315
316	/**
317	* VMX VM-exit handler.
318	*
319	* @returns VBox status code.
320	* @param pVCpu Pointer to the VMCPU.
321	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
322	* out-of-sync. Make sure to update the required
323	* fields before using them.
324	* @param pVmxTransient Pointer to the VMX-transient structure.
325	*/
326	#ifndef HMVMX_USE_FUNCTION_TABLE
327	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
328	#else
329	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330	/** Pointer to VM-exit handler. */
331	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
332	#endif
333
334
335	/*******************************************************************************
336	* Internal Functions *
337	*******************************************************************************/
338	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
339	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
340	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
341	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
342	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
343	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
344	#endif
345	#ifndef HMVMX_USE_FUNCTION_TABLE
346	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
347	# define HMVMX_EXIT_DECL static int
348	#else
349	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
350	#endif
351
352	/** @name VM-exit handlers.
353	* @{
354	*/
355	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
356	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
357	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
358	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
359	static FNVMXEXITHANDLER hmR0VmxExitSipi;
360	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitSmi;
362	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
364	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
365	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
366	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
367	static FNVMXEXITHANDLER hmR0VmxExitHlt;
368	static FNVMXEXITHANDLER hmR0VmxExitInvd;
369	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
370	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
371	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
372	static FNVMXEXITHANDLER hmR0VmxExitRsm;
373	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
374	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
375	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
376	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
377	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
379	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
380	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
381	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
382	static FNVMXEXITHANDLER hmR0VmxExitMwait;
383	static FNVMXEXITHANDLER hmR0VmxExitMtf;
384	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
385	static FNVMXEXITHANDLER hmR0VmxExitPause;
386	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
387	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
388	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
391	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
392	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
393	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
394	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
395	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
396	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
397	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
398	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
399	/** @} */
400
401	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
408	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409	#endif
410	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
411
412	/*******************************************************************************
413	* Global Variables *
414	*******************************************************************************/
415	#ifdef HMVMX_USE_FUNCTION_TABLE
416
417	/**
418	* VMX_EXIT dispatch table.
419	*/
420	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
421	{
422	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
423	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
424	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
425	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
426	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
427	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
428	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
429	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
430	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
431	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
432	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
433	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
434	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
435	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
436	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
437	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
438	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
439	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
440	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
441	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
442	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
443	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
444	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
445	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
446	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
447	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
448	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
449	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
450	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
451	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
452	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
453	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
454	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
455	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
456	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
457	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
458	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
459	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
460	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
461	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
462	/* 40 UNDEFINED */ hmR0VmxExitPause,
463	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
464	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
465	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
466	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
467	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
468	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
470	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
471	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
472	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
473	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
474	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
475	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
476	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
477	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
478	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
479	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
480	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
481	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
482	};
483	#endif /* HMVMX_USE_FUNCTION_TABLE */
484
485	#ifdef VBOX_STRICT
486	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
487	{
488	/* 0 */ "(Not Used)",
489	/* 1 */ "VMCALL executed in VMX root operation.",
490	/* 2 */ "VMCLEAR with invalid physical address.",
491	/* 3 */ "VMCLEAR with VMXON pointer.",
492	/* 4 */ "VMLAUNCH with non-clear VMCS.",
493	/* 5 */ "VMRESUME with non-launched VMCS.",
494	/* 6 */ "VMRESUME after VMXOFF",
495	/* 7 */ "VM entry with invalid control fields.",
496	/* 8 */ "VM entry with invalid host state fields.",
497	/* 9 */ "VMPTRLD with invalid physical address.",
498	/* 10 */ "VMPTRLD with VMXON pointer.",
499	/* 11 */ "VMPTRLD with incorrect revision identifier.",
500	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
501	/* 13 */ "VMWRITE to read-only VMCS component.",
502	/* 14 */ "(Not Used)",
503	/* 15 */ "VMXON executed in VMX root operation.",
504	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
505	/* 17 */ "VM entry with non-launched executing VMCS.",
506	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
507	/* 19 */ "VMCALL with non-clear VMCS.",
508	/* 20 */ "VMCALL with invalid VM-exit control fields.",
509	/* 21 */ "(Not Used)",
510	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
511	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
512	/* 24 */ "VMCALL with invalid SMM-monitor features.",
513	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
514	/* 26 */ "VM entry with events blocked by MOV SS.",
515	/* 27 */ "(Not Used)",
516	/* 28 */ "Invalid operand to INVEPT/INVVPID."
517	};
518	#endif /* VBOX_STRICT */
519
520
521
522	/**
523	* Updates the VM's last error record. If there was a VMX instruction error,
524	* reads the error data from the VMCS and updates VCPU's last error record as
525	* well.
526	*
527	* @param pVM Pointer to the VM.
528	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
529	* VERR_VMX_UNABLE_TO_START_VM or
530	* VERR_VMX_INVALID_VMCS_FIELD).
531	* @param rc The error code.
532	*/
533	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
534	{
535	AssertPtr(pVM);
536	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
537	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
538	{
539	AssertPtrReturnVoid(pVCpu);
540	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
541	}
542	pVM->hm.s.lLastError = rc;
543	}
544
545
546	/**
547	* Reads the VM-entry interruption-information field from the VMCS into the VMX
548	* transient structure.
549	*
550	* @returns VBox status code.
551	* @param pVmxTransient Pointer to the VMX transient structure.
552	*
553	* @remarks No-long-jump zone!!!
554	*/
555	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
556	{
557	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
558	AssertRCReturn(rc, rc);
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Reads the VM-entry exception error code field from the VMCS into
565	* the VMX transient structure.
566	*
567	* @returns VBox status code.
568	* @param pVmxTransient Pointer to the VMX transient structure.
569	*
570	* @remarks No-long-jump zone!!!
571	*/
572	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
573	{
574	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
575	AssertRCReturn(rc, rc);
576	return VINF_SUCCESS;
577	}
578
579
580	/**
581	* Reads the VM-entry exception error code field from the VMCS into
582	* the VMX transient structure.
583	*
584	* @returns VBox status code.
585	* @param pVmxTransient Pointer to the VMX transient structure.
586	*
587	* @remarks No-long-jump zone!!!
588	*/
589	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
590	{
591	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
592	AssertRCReturn(rc, rc);
593	return VINF_SUCCESS;
594	}
595
596
597	/**
598	* Reads the VM-exit interruption-information field from the VMCS into the VMX
599	* transient structure.
600	*
601	* @returns VBox status code.
602	* @param pVmxTransient Pointer to the VMX transient structure.
603	*/
604	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
605	{
606	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
607	{
608	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
609	AssertRCReturn(rc, rc);
610	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
611	}
612	return VINF_SUCCESS;
613	}
614
615
616	/**
617	* Reads the VM-exit interruption error code from the VMCS into the VMX
618	* transient structure.
619	*
620	* @returns VBox status code.
621	* @param pVmxTransient Pointer to the VMX transient structure.
622	*/
623	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
624	{
625	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
626	{
627	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
628	AssertRCReturn(rc, rc);
629	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
630	}
631	return VINF_SUCCESS;
632	}
633
634
635	/**
636	* Reads the VM-exit instruction length field from the VMCS into the VMX
637	* transient structure.
638	*
639	* @returns VBox status code.
640	* @param pVCpu Pointer to the VMCPU.
641	* @param pVmxTransient Pointer to the VMX transient structure.
642	*/
643	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
644	{
645	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
646	{
647	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
648	AssertRCReturn(rc, rc);
649	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
650	}
651	return VINF_SUCCESS;
652	}
653
654
655	/**
656	* Reads the VM-exit instruction-information field from the VMCS into
657	* the VMX transient structure.
658	*
659	* @returns VBox status code.
660	* @param pVmxTransient Pointer to the VMX transient structure.
661	*/
662	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
663	{
664	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
665	{
666	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
667	AssertRCReturn(rc, rc);
668	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
669	}
670	return VINF_SUCCESS;
671	}
672
673
674	/**
675	* Reads the exit qualification from the VMCS into the VMX transient structure.
676	*
677	* @returns VBox status code.
678	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
679	* case).
680	* @param pVmxTransient Pointer to the VMX transient structure.
681	*/
682	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
683	{
684	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
685	{
686	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
687	AssertRCReturn(rc, rc);
688	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
689	}
690	return VINF_SUCCESS;
691	}
692
693
694	/**
695	* Reads the IDT-vectoring information field from the VMCS into the VMX
696	* transient structure.
697	*
698	* @returns VBox status code.
699	* @param pVmxTransient Pointer to the VMX transient structure.
700	*
701	* @remarks No-long-jump zone!!!
702	*/
703	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
704	{
705	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
706	{
707	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
708	AssertRCReturn(rc, rc);
709	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
710	}
711	return VINF_SUCCESS;
712	}
713
714
715	/**
716	* Reads the IDT-vectoring error code from the VMCS into the VMX
717	* transient structure.
718	*
719	* @returns VBox status code.
720	* @param pVmxTransient Pointer to the VMX transient structure.
721	*/
722	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
723	{
724	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
725	{
726	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
727	AssertRCReturn(rc, rc);
728	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
729	}
730	return VINF_SUCCESS;
731	}
732
733
734	/**
735	* Enters VMX root mode operation on the current CPU.
736	*
737	* @returns VBox status code.
738	* @param pVM Pointer to the VM (optional, can be NULL, after
739	* a resume).
740	* @param HCPhysCpuPage Physical address of the VMXON region.
741	* @param pvCpuPage Pointer to the VMXON region.
742	*/
743	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
744	{
745	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
746	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
747	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
748
749	if (pVM)
750	{
751	/* Write the VMCS revision dword to the VMXON region. */
752	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
753	}
754
755	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
756	RTCCUINTREG uEflags = ASMIntDisableFlags();
757
758	/* Enable the VMX bit in CR4 if necessary. */
759	RTCCUINTREG uCr4 = ASMGetCR4();
760	if (!(uCr4 & X86_CR4_VMXE))
761	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
762
763	/* Enter VMX root mode. */
764	int rc = VMXEnable(HCPhysCpuPage);
765	if (RT_FAILURE(rc))
766	ASMSetCR4(uCr4);
767
768	/* Restore interrupts. */
769	ASMSetFlags(uEflags);
770	return rc;
771	}
772
773
774	/**
775	* Exits VMX root mode operation on the current CPU.
776	*
777	* @returns VBox status code.
778	*/
779	static int hmR0VmxLeaveRootMode(void)
780	{
781	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
782
783	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
784	RTCCUINTREG uEflags = ASMIntDisableFlags();
785
786	/* If we're for some reason not in VMX root mode, then don't leave it. */
787	RTCCUINTREG uHostCR4 = ASMGetCR4();
788
789	int rc;
790	if (uHostCR4 & X86_CR4_VMXE)
791	{
792	/* Exit VMX root mode and clear the VMX bit in CR4. */
793	VMXDisable();
794	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
795	rc = VINF_SUCCESS;
796	}
797	else
798	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
799
800	/* Restore interrupts. */
801	ASMSetFlags(uEflags);
802	return rc;
803	}
804
805
806	/**
807	* Allocates and maps one physically contiguous page. The allocated page is
808	* zero'd out. (Used by various VT-x structures).
809	*
810	* @returns IPRT status code.
811	* @param pMemObj Pointer to the ring-0 memory object.
812	* @param ppVirt Where to store the virtual address of the
813	* allocation.
814	* @param pPhys Where to store the physical address of the
815	* allocation.
816	*/
817	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
818	{
819	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
821	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
822
823	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
824	if (RT_FAILURE(rc))
825	return rc;
826	ppVirt = RTR0MemObjAddress(pMemObj);
827	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
828	ASMMemZero32(*ppVirt, PAGE_SIZE);
829	return VINF_SUCCESS;
830	}
831
832
833	/**
834	* Frees and unmaps an allocated physical page.
835	*
836	* @param pMemObj Pointer to the ring-0 memory object.
837	* @param ppVirt Where to re-initialize the virtual address of
838	* allocation as 0.
839	* @param pHCPhys Where to re-initialize the physical address of the
840	* allocation as 0.
841	*/
842	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
843	{
844	AssertPtr(pMemObj);
845	AssertPtr(ppVirt);
846	AssertPtr(pHCPhys);
847	if (*pMemObj != NIL_RTR0MEMOBJ)
848	{
849	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
850	AssertRC(rc);
851	*pMemObj = NIL_RTR0MEMOBJ;
852	*ppVirt = 0;
853	*pHCPhys = 0;
854	}
855	}
856
857
858	/**
859	* Worker function to free VT-x related structures.
860	*
861	* @returns IPRT status code.
862	* @param pVM Pointer to the VM.
863	*/
864	static void hmR0VmxStructsFree(PVM pVM)
865	{
866	for (VMCPUID i = 0; i < pVM->cCpus; i++)
867	{
868	PVMCPU pVCpu = &pVM->aCpus[i];
869	AssertPtr(pVCpu);
870
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
872	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873
874	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
875	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
876
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
878	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
879	}
880
881	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
882	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
883	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
884	#endif
885	}
886
887
888	/**
889	* Worker function to allocate VT-x related VM structures.
890	*
891	* @returns IPRT status code.
892	* @param pVM Pointer to the VM.
893	*/
894	static int hmR0VmxStructsAlloc(PVM pVM)
895	{
896	/*
897	* Initialize members up-front so we can cleanup properly on allocation failure.
898	*/
899	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
900	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
901	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
902	pVM->hm.s.vmx.HCPhys##a_Name = 0;
903
904	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
905	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
908
909	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
910	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
911	#endif
912	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
913
914	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
915	for (VMCPUID i = 0; i < pVM->cCpus; i++)
916	{
917	PVMCPU pVCpu = &pVM->aCpus[i];
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
922	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
923	}
924	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
925	#undef VMXLOCAL_INIT_VM_MEMOBJ
926
927	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
928	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
929	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
930	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
931
932	/*
933	* Allocate all the VT-x structures.
934	*/
935	int rc = VINF_SUCCESS;
936	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
937	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
938	if (RT_FAILURE(rc))
939	goto cleanup;
940	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
941	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
942	#endif
943
944	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
945	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
946	{
947	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
948	&pVM->hm.s.vmx.HCPhysApicAccess);
949	if (RT_FAILURE(rc))
950	goto cleanup;
951	}
952
953	/*
954	* Initialize per-VCPU VT-x structures.
955	*/
956	for (VMCPUID i = 0; i < pVM->cCpus; i++)
957	{
958	PVMCPU pVCpu = &pVM->aCpus[i];
959	AssertPtr(pVCpu);
960
961	/* Allocate the VM control structure (VMCS). */
962	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
963	if (RT_FAILURE(rc))
964	goto cleanup;
965
966	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
967	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
968	{
969	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
970	&pVCpu->hm.s.vmx.HCPhysVirtApic);
971	if (RT_FAILURE(rc))
972	goto cleanup;
973	}
974
975	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
976	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
977	{
978	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
979	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
980	if (RT_FAILURE(rc))
981	goto cleanup;
982	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
983	}
984
985	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
986	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
987	if (RT_FAILURE(rc))
988	goto cleanup;
989
990	/* Allocate the VM-exit MSR-load page for the host MSRs. */
991	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
992	if (RT_FAILURE(rc))
993	goto cleanup;
994	}
995
996	return VINF_SUCCESS;
997
998	cleanup:
999	hmR0VmxStructsFree(pVM);
1000	return rc;
1001	}
1002
1003
1004	/**
1005	* Does global VT-x initialization (called during module initialization).
1006	*
1007	* @returns VBox status code.
1008	*/
1009	VMMR0DECL(int) VMXR0GlobalInit(void)
1010	{
1011	#ifdef HMVMX_USE_FUNCTION_TABLE
1012	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1013	# ifdef VBOX_STRICT
1014	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1015	Assert(g_apfnVMExitHandlers[i]);
1016	# endif
1017	#endif
1018	return VINF_SUCCESS;
1019	}
1020
1021
1022	/**
1023	* Does global VT-x termination (called during module termination).
1024	*/
1025	VMMR0DECL(void) VMXR0GlobalTerm()
1026	{
1027	/* Nothing to do currently. */
1028	}
1029
1030
1031	/**
1032	* Sets up and activates VT-x on the current CPU.
1033	*
1034	* @returns VBox status code.
1035	* @param pCpu Pointer to the global CPU info struct.
1036	* @param pVM Pointer to the VM (can be NULL after a host resume
1037	* operation).
1038	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1039	* fEnabledByHost is true).
1040	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1041	* @a fEnabledByHost is true).
1042	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1043	* enable VT-x on the host.
1044	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1045	*/
1046	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1047	void *pvMsrs)
1048	{
1049	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1050	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1051	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1052
1053	/* Enable VT-x if it's not already enabled by the host. */
1054	if (!fEnabledByHost)
1055	{
1056	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1057	if (RT_FAILURE(rc))
1058	return rc;
1059	}
1060
1061	/*
1062	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1063	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1064	*/
1065	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1066	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1067	{
1068	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1069	pCpu->fFlushAsidBeforeUse = false;
1070	}
1071	else
1072	pCpu->fFlushAsidBeforeUse = true;
1073
1074	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1075	++pCpu->cTlbFlushes;
1076
1077	return VINF_SUCCESS;
1078	}
1079
1080
1081	/**
1082	* Deactivates VT-x on the current CPU.
1083	*
1084	* @returns VBox status code.
1085	* @param pCpu Pointer to the global CPU info struct.
1086	* @param pvCpuPage Pointer to the VMXON region.
1087	* @param HCPhysCpuPage Physical address of the VMXON region.
1088	*
1089	* @remarks This function should never be called when SUPR0EnableVTx() or
1090	* similar was used to enable VT-x on the host.
1091	*/
1092	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1093	{
1094	NOREF(pCpu);
1095	NOREF(pvCpuPage);
1096	NOREF(HCPhysCpuPage);
1097
1098	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1099	return hmR0VmxLeaveRootMode();
1100	}
1101
1102
1103	/**
1104	* Sets the permission bits for the specified MSR in the MSR bitmap.
1105	*
1106	* @param pVCpu Pointer to the VMCPU.
1107	* @param uMSR The MSR value.
1108	* @param enmRead Whether reading this MSR causes a VM-exit.
1109	* @param enmWrite Whether writing this MSR causes a VM-exit.
1110	*/
1111	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1112	{
1113	int32_t iBit;
1114	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1115
1116	/*
1117	* Layout:
1118	* 0x000 - 0x3ff - Low MSR read bits
1119	* 0x400 - 0x7ff - High MSR read bits
1120	* 0x800 - 0xbff - Low MSR write bits
1121	* 0xc00 - 0xfff - High MSR write bits
1122	*/
1123	if (uMsr <= 0x00001FFF)
1124	iBit = uMsr;
1125	else if ( uMsr >= 0xC0000000
1126	&& uMsr <= 0xC0001FFF)
1127	{
1128	iBit = (uMsr - 0xC0000000);
1129	pbMsrBitmap += 0x400;
1130	}
1131	else
1132	{
1133	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1134	return;
1135	}
1136
1137	Assert(iBit <= 0x1fff);
1138	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1139	ASMBitSet(pbMsrBitmap, iBit);
1140	else
1141	ASMBitClear(pbMsrBitmap, iBit);
1142
1143	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1144	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1145	else
1146	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1147	}
1148
1149
1150	#ifdef VBOX_STRICT
1151	/**
1152	* Gets the permission bits for the specified MSR in the MSR bitmap.
1153	*
1154	* @returns VBox status code.
1155	* @retval VINF_SUCCESS if the specified MSR is found.
1156	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1157	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1158	*
1159	* @param pVCpu Pointer to the VMCPU.
1160	* @param uMsr The MSR.
1161	* @param penmRead Where to store the read permissions.
1162	* @param penmWrite Where to store the write permissions.
1163	*/
1164	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1165	{
1166	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1167	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1168	int32_t iBit;
1169	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1170
1171	/* See hmR0VmxSetMsrPermission() for the layout. */
1172	if (uMsr <= 0x00001FFF)
1173	iBit = uMsr;
1174	else if ( uMsr >= 0xC0000000
1175	&& uMsr <= 0xC0001FFF)
1176	{
1177	iBit = (uMsr - 0xC0000000);
1178	pbMsrBitmap += 0x400;
1179	}
1180	else
1181	{
1182	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1183	return VERR_NOT_SUPPORTED;
1184	}
1185
1186	Assert(iBit <= 0x1fff);
1187	if (ASMBitTest(pbMsrBitmap, iBit))
1188	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1189	else
1190	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1191
1192	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1193	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1194	else
1195	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1196	return VINF_SUCCESS;
1197	}
1198	#endif /* VBOX_STRICT */
1199
1200
1201	/**
1202	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1203	* area.
1204	*
1205	* @returns VBox status code.
1206	* @param pVCpu Pointer to the VMCPU.
1207	* @param cMsrs The number of MSRs.
1208	*/
1209	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1210	{
1211	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1212	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1213	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1214	{
1215	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1216	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1217	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1218	}
1219
1220	/* Update number of guest MSRs to load/store across the world-switch. */
1221	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1223
1224	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1225	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1226
1227	/* Update the VCPU's copy of the MSR count. */
1228	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1229
1230	return VINF_SUCCESS;
1231	}
1232
1233
1234	/**
1235	* Adds a new (or updates the value of an existing) guest/host MSR
1236	* pair to be swapped during the world-switch as part of the
1237	* auto-load/store MSR area in the VMCS.
1238	*
1239	* @returns VBox status code.
1240	* @param pVCpu Pointer to the VMCPU.
1241	* @param uMsr The MSR.
1242	* @param uGuestMsr Value of the guest MSR.
1243	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1244	* necessary.
1245	*/
1246	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1247	{
1248	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1249	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1250	uint32_t i;
1251	for (i = 0; i < cMsrs; i++)
1252	{
1253	if (pGuestMsr->u32Msr == uMsr)
1254	break;
1255	pGuestMsr++;
1256	}
1257
1258	bool fAdded = false;
1259	if (i == cMsrs)
1260	{
1261	++cMsrs;
1262	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1263	AssertRCReturn(rc, rc);
1264
1265	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1266	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1267	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1268
1269	fAdded = true;
1270	}
1271
1272	/* Update the MSR values in the auto-load/store MSR area. */
1273	pGuestMsr->u32Msr = uMsr;
1274	pGuestMsr->u64Value = uGuestMsrValue;
1275
1276	/* Create/update the MSR slot in the host MSR area. */
1277	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1278	pHostMsr += i;
1279	pHostMsr->u32Msr = uMsr;
1280
1281	/*
1282	* Update the host MSR only when requested by the caller AND when we're
1283	* adding it to the auto-load/store area. Otherwise, it would have been
1284	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1285	*/
1286	if ( fAdded
1287	&& fUpdateHostMsr)
1288	{
1289	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1290	}
1291
1292	return VINF_SUCCESS;
1293	}
1294
1295
1296	/**
1297	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1298	* auto-load/store MSR area in the VMCS.
1299	*
1300	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1301	* area.
1302	*
1303	* @returns VBox status code.
1304	* @param pVCpu Pointer to the VMCPU.
1305	* @param uMsr The MSR.
1306	*/
1307	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1308	{
1309	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1310	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1311	for (uint32_t i = 0; i < cMsrs; i++)
1312	{
1313	/* Find the MSR. */
1314	if (pGuestMsr->u32Msr == uMsr)
1315	{
1316	/* If it's the last MSR, simply reduce the count. */
1317	if (i == cMsrs - 1)
1318	{
1319	--cMsrs;
1320	break;
1321	}
1322
1323	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1324	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1325	pLastGuestMsr += cMsrs;
1326	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1327	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1328
1329	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1330	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	pLastHostMsr += cMsrs;
1332	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1333	pHostMsr->u64Value = pLastHostMsr->u64Value;
1334	--cMsrs;
1335	break;
1336	}
1337	pGuestMsr++;
1338	}
1339
1340	/* Update the VMCS if the count changed (meaning the MSR was found). */
1341	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1342	{
1343	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1344	AssertRCReturn(rc, rc);
1345
1346	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1347	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1348	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1349	}
1350
1351	return VINF_SUCCESS;
1352	}
1353
1354
1355	/**
1356	* Checks if the specified guest MSR is part of the auto-load/store area in
1357	* the VMCS.
1358	*
1359	* @returns true if found, false otherwise.
1360	* @param pVCpu Pointer to the VMCPU.
1361	* @param uMsr The MSR to find.
1362	*/
1363	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1364	{
1365	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1366	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1367
1368	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1369	{
1370	if (pGuestMsr->u32Msr == uMsr)
1371	return true;
1372	}
1373	return false;
1374	}
1375
1376
1377	/**
1378	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1379	*
1380	* @param pVCpu Pointer to the VMCPU.
1381	*
1382	* @remarks No-long-jump zone!!!
1383	*/
1384	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1385	{
1386	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1387	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1388	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1389	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1390
1391	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1392	{
1393	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1394	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1395	}
1396
1397	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1398	}
1399
1400
1401	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1402	/**
1403	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1404	* perform lazy restoration of the host MSRs while leaving VT-x.
1405	*
1406	* @param pVCpu Pointer to the VMCPU.
1407	*
1408	* @remarks No-long-jump zone!!!
1409	*/
1410	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1411	{
1412	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1413
1414	#define VMXLOCAL_SAVE_HOST_MSR(uMsr, a_HostMsrField, RestoreFlag) \
1415	do { \
1416	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag))) \
1417	{ \
1418	pVCpu->hm.s.vmx.u64Host##a_HostMsrField = ASMRdMsr(uMsr); \
1419	} \
1420	} while (0)
1421
1422	/*
1423	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1424	*/
1425	VMXLOCAL_SAVE_HOST_MSR(MSR_K8_LSTAR, LStarMsr, VMX_RESTORE_HOST_MSR_LSTAR);
1426	VMXLOCAL_SAVE_HOST_MSR(MSR_K6_STAR, StarMsr, VMX_RESTORE_HOST_MSR_STAR);
1427	VMXLOCAL_SAVE_HOST_MSR(MSR_K8_SF_MASK, SFMaskMsr, VMX_RESTORE_HOST_MSR_SFMASK);
1428	VMXLOCAL_SAVE_HOST_MSR(MSR_K8_KERNEL_GS_BASE, KernelGSBaseMsr, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1429
1430	#undef VMXLOCAL_SAVE_HOST_MSR
1431	}
1432
1433
1434	/**
1435	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1436	* lazily while leaving VT-x.
1437	*
1438	* @returns true if it does, false otherwise.
1439	* @param pVCpu Pointer to the VMCPU.
1440	* @param uMsr The MSR to check.
1441	*/
1442	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1443	{
1444	NOREF(pVCpu);
1445	switch (uMsr)
1446	{
1447	case MSR_K8_LSTAR:
1448	case MSR_K6_STAR:
1449	case MSR_K8_SF_MASK:
1450	case MSR_K8_KERNEL_GS_BASE:
1451	return true;
1452	}
1453	return false;
1454	}
1455
1456
1457	/**
1458	* Saves a set of guests MSRs back into the guest-CPU context.
1459	*
1460	* @param pVCpu Pointer to the VMCPU.
1461	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1462	* out-of-sync. Make sure to update the required fields
1463	* before using them.
1464	*
1465	* @remarks No-long-jump zone!!!
1466	*/
1467	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1468	{
1469	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1470	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1471
1472	#define VMXLOCAL_SAVE_GUEST_MSR(uMsr, a_GuestMsrField, RestoreFlag) \
1473	do { \
1474	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag)) \
1475	{ \
1476	pMixedCtx->msr##a_GuestMsrField = ASMRdMsr(uMsr); \
1477	Log4(("hmR0VmxLazySaveGuestMsrs: uMsr=%#RX32 GuestValue=%#RX64\n", (uMsr), pMixedCtx->msr##a_GuestMsrField)); \
1478	} \
1479	} while (0)
1480
1481	VMXLOCAL_SAVE_GUEST_MSR(MSR_K8_LSTAR, LSTAR, VMX_RESTORE_HOST_MSR_LSTAR);
1482	VMXLOCAL_SAVE_GUEST_MSR(MSR_K6_STAR, STAR, VMX_RESTORE_HOST_MSR_STAR);
1483	VMXLOCAL_SAVE_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, VMX_RESTORE_HOST_MSR_SFMASK);
1484	VMXLOCAL_SAVE_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1485
1486	#undef VMXLOCAL_SAVE_GUEST_MSR
1487	}
1488
1489
1490	/**
1491	* Loads a set of guests MSRs to allow read/passthru to the guest.
1492	*
1493	* The name of this function is slightly confusing. This function does NOT
1494	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1495	* common prefix for functions dealing with "lazy restoration" of the shared
1496	* MSRs.
1497	*
1498	* @param pVCpu Pointer to the VMCPU.
1499	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1500	* out-of-sync. Make sure to update the required fields
1501	* before using them.
1502	*
1503	* @remarks No-long-jump zone!!!
1504	*/
1505	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1506	{
1507	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1508	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1509
1510	#define VMXLOCAL_LOAD_GUEST_MSR(uMsr, a_GuestMsrField, a_HostMsrField, RestoreFlag) \
1511	do { \
1512	if ( (pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag)) \
1513	\|\| pMixedCtx->msr##a_GuestMsrField != pVCpu->hm.s.vmx.u64Host##a_HostMsrField) \
1514	{ \
1515	ASMWrMsr((uMsr), pMixedCtx->msr##a_GuestMsrField); \
1516	} \
1517	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= (RestoreFlag); \
1518	Log4(("Load: MSRSWAP uMsr=%#RX32 GuestValue=%#RX64\n", (uMsr), pMixedCtx->msr##a_GuestMsrField)); \
1519	} while (0)
1520
1521	VMXLOCAL_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStarMsr, VMX_RESTORE_HOST_MSR_LSTAR);
1522	VMXLOCAL_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, StarMsr, VMX_RESTORE_HOST_MSR_STAR);
1523	VMXLOCAL_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMaskMsr, VMX_RESTORE_HOST_MSR_SFMASK);
1524	VMXLOCAL_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBaseMsr, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1525
1526	#undef VMXLOCAL_LOAD_GUEST_MSR
1527	}
1528
1529
1530	/**
1531	* Performs lazy restoration of the set of host MSRs if they were previously
1532	* loaded with guest MSR values.
1533	*
1534	* @param pVCpu Pointer to the VMCPU.
1535	*
1536	* @remarks No-long-jump zone!!!
1537	* @remarks The guest MSRs should have been saved back into the guest-CPU
1538	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1539	*/
1540	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1541	{
1542	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1543	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1544
1545	#define VMXLOCAL_RESTORE_HOST_MSR(uMsr, a_HostMsrField, RestoreFlag) \
1546	do { \
1547	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag)) \
1548	{ \
1549	ASMWrMsr((uMsr), pVCpu->hm.s.vmx.u64Host##a_HostMsrField); \
1550	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(RestoreFlag); \
1551	Log4(("hmR0VmxLazyRestoreHostMsrs: uMsr=%#RX32 HostValue=%#RX64\n", (uMsr), \
1552	pVCpu->hm.s.vmx.u64Host##a_HostMsrField)); \
1553	} \
1554	} while (0)
1555
1556	VMXLOCAL_RESTORE_HOST_MSR(MSR_K8_LSTAR, LStarMsr, VMX_RESTORE_HOST_MSR_LSTAR);
1557	VMXLOCAL_RESTORE_HOST_MSR(MSR_K6_STAR, StarMsr, VMX_RESTORE_HOST_MSR_STAR);
1558	VMXLOCAL_RESTORE_HOST_MSR(MSR_K8_SF_MASK, SFMaskMsr, VMX_RESTORE_HOST_MSR_SFMASK);
1559	VMXLOCAL_RESTORE_HOST_MSR(MSR_K8_KERNEL_GS_BASE, KernelGSBaseMsr, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1560
1561	#undef VMXLOCAL_RESTORE_HOST_MSR
1562	}
1563	#endif /* HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
1564
1565
1566	#ifdef VBOX_STRICT
1567	/**
1568	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1569	* VMCS are correct.
1570	*
1571	* @param pVCpu Pointer to the VMCPU.
1572	*/
1573	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1574	{
1575	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1576
1577	/* Verify MSR counts in the VMCS are what we think it should be. */
1578	uint32_t cMsrs;
1579	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1580	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1581
1582	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1583	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1584
1585	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1586	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1587
1588	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1589	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1590	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1591	{
1592	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1593	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1594	pGuestMsr->u32Msr));
1595
1596	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1597	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1598	pHostMsr->u64Value, u64Msr));
1599
1600	/* Verify that the permissions are as expected in the MSR bitmap. */
1601	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1602	{
1603	VMXMSREXITREAD enmRead;
1604	VMXMSREXITWRITE enmWrite;
1605	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1606	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1607	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1608	pGuestMsr->u32Msr));
1609	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1610	pGuestMsr->u32Msr));
1611	}
1612	}
1613	}
1614	# endif /* VBOX_STRICT */
1615
1616
1617	/**
1618	* Flushes the TLB using EPT.
1619	*
1620	* @returns VBox status code.
1621	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1622	* enmFlush).
1623	* @param enmFlush Type of flush.
1624	*
1625	* @remarks Caller is responsible for making sure this function is called only
1626	* when NestedPaging is supported and providing @a enmFlush that is
1627	* supported by the CPU.
1628	* @remarks Can be called with interrupts disabled.
1629	*/
1630	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1631	{
1632	uint64_t au64Descriptor[2];
1633	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1634	au64Descriptor[0] = 0;
1635	else
1636	{
1637	Assert(pVCpu);
1638	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1639	}
1640	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1641
1642	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1643	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1644	rc));
1645	if ( RT_SUCCESS(rc)
1646	&& pVCpu)
1647	{
1648	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1649	}
1650	}
1651
1652
1653	/**
1654	* Flushes the TLB using VPID.
1655	*
1656	* @returns VBox status code.
1657	* @param pVM Pointer to the VM.
1658	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1659	* enmFlush).
1660	* @param enmFlush Type of flush.
1661	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1662	* on @a enmFlush).
1663	*
1664	* @remarks Can be called with interrupts disabled.
1665	*/
1666	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1667	{
1668	NOREF(pVM);
1669	AssertPtr(pVM);
1670	Assert(pVM->hm.s.vmx.fVpid);
1671
1672	uint64_t au64Descriptor[2];
1673	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1674	{
1675	au64Descriptor[0] = 0;
1676	au64Descriptor[1] = 0;
1677	}
1678	else
1679	{
1680	AssertPtr(pVCpu);
1681	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1682	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1683	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1684	au64Descriptor[1] = GCPtr;
1685	}
1686
1687	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1688	AssertMsg(rc == VINF_SUCCESS,
1689	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1690	if ( RT_SUCCESS(rc)
1691	&& pVCpu)
1692	{
1693	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1694	}
1695	}
1696
1697
1698	/**
1699	* Invalidates a guest page by guest virtual address. Only relevant for
1700	* EPT/VPID, otherwise there is nothing really to invalidate.
1701	*
1702	* @returns VBox status code.
1703	* @param pVM Pointer to the VM.
1704	* @param pVCpu Pointer to the VMCPU.
1705	* @param GCVirt Guest virtual address of the page to invalidate.
1706	*/
1707	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1708	{
1709	AssertPtr(pVM);
1710	AssertPtr(pVCpu);
1711	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1712
1713	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1714	if (!fFlushPending)
1715	{
1716	/*
1717	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1718	* See @bugref{6043} and @bugref{6177}.
1719	*
1720	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1721	* function maybe called in a loop with individual addresses.
1722	*/
1723	if (pVM->hm.s.vmx.fVpid)
1724	{
1725	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1726	{
1727	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1728	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1729	}
1730	else
1731	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1732	}
1733	else if (pVM->hm.s.fNestedPaging)
1734	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1735	}
1736
1737	return VINF_SUCCESS;
1738	}
1739
1740
1741	/**
1742	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1743	* otherwise there is nothing really to invalidate.
1744	*
1745	* @returns VBox status code.
1746	* @param pVM Pointer to the VM.
1747	* @param pVCpu Pointer to the VMCPU.
1748	* @param GCPhys Guest physical address of the page to invalidate.
1749	*/
1750	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1751	{
1752	NOREF(pVM); NOREF(GCPhys);
1753	LogFlowFunc(("%RGp\n", GCPhys));
1754
1755	/*
1756	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1757	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1758	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1759	*/
1760	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1761	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1762	return VINF_SUCCESS;
1763	}
1764
1765
1766	/**
1767	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1768	* case where neither EPT nor VPID is supported by the CPU.
1769	*
1770	* @param pVM Pointer to the VM.
1771	* @param pVCpu Pointer to the VMCPU.
1772	* @param pCpu Pointer to the global HM struct.
1773	*
1774	* @remarks Called with interrupts disabled.
1775	*/
1776	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1777	{
1778	AssertPtr(pVCpu);
1779	AssertPtr(pCpu);
1780	NOREF(pVM);
1781
1782	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1783
1784	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1785	#if 0
1786	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1787	pVCpu->hm.s.TlbShootdown.cPages = 0;
1788	#endif
1789
1790	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1791	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1792	pVCpu->hm.s.fForceTLBFlush = false;
1793	return;
1794	}
1795
1796
1797	/**
1798	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1799	*
1800	* @param pVM Pointer to the VM.
1801	* @param pVCpu Pointer to the VMCPU.
1802	* @param pCpu Pointer to the global HM CPU struct.
1803	* @remarks All references to "ASID" in this function pertains to "VPID" in
1804	* Intel's nomenclature. The reason is, to avoid confusion in compare
1805	* statements since the host-CPU copies are named "ASID".
1806	*
1807	* @remarks Called with interrupts disabled.
1808	*/
1809	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1810	{
1811	#ifdef VBOX_WITH_STATISTICS
1812	bool fTlbFlushed = false;
1813	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1814	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1815	if (!fTlbFlushed) \
1816	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1817	} while (0)
1818	#else
1819	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1820	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1821	#endif
1822
1823	AssertPtr(pVM);
1824	AssertPtr(pCpu);
1825	AssertPtr(pVCpu);
1826	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1827	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1828	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1829
1830
1831	/*
1832	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1833	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1834	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1835	*/
1836	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1837	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1838	{
1839	++pCpu->uCurrentAsid;
1840	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1841	{
1842	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1843	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1844	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1845	}
1846
1847	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1848	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1849	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1850
1851	/*
1852	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1853	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1854	*/
1855	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1856	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1857	HMVMX_SET_TAGGED_TLB_FLUSHED();
1858	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1859	}
1860
1861	/* Check for explicit TLB shootdowns. */
1862	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1863	{
1864	/*
1865	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1866	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1867	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1868	* but not guest-physical mappings.
1869	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1870	*/
1871	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1872	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1873	HMVMX_SET_TAGGED_TLB_FLUSHED();
1874	}
1875
1876	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1877	* where it is commented out. Support individual entry flushing
1878	* someday. */
1879	#if 0
1880	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1881	{
1882	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1883
1884	/*
1885	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1886	* as supported by the CPU.
1887	*/
1888	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1889	{
1890	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1891	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1892	}
1893	else
1894	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1895
1896	HMVMX_SET_TAGGED_TLB_FLUSHED();
1897	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1898	pVCpu->hm.s.TlbShootdown.cPages = 0;
1899	}
1900	#endif
1901
1902	pVCpu->hm.s.fForceTLBFlush = false;
1903
1904	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1905
1906	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1907	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1908	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1909	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1910	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1911	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1912	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1913	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1914
1915	/* Update VMCS with the VPID. */
1916	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1917	AssertRC(rc);
1918
1919	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1920	}
1921
1922
1923	/**
1924	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1925	*
1926	* @returns VBox status code.
1927	* @param pVM Pointer to the VM.
1928	* @param pVCpu Pointer to the VMCPU.
1929	* @param pCpu Pointer to the global HM CPU struct.
1930	*
1931	* @remarks Called with interrupts disabled.
1932	*/
1933	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1934	{
1935	AssertPtr(pVM);
1936	AssertPtr(pVCpu);
1937	AssertPtr(pCpu);
1938	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1939	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1940
1941	/*
1942	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1943	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1944	*/
1945	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1946	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1947	{
1948	pVCpu->hm.s.fForceTLBFlush = true;
1949	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1950	}
1951
1952	/* Check for explicit TLB shootdown flushes. */
1953	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1954	{
1955	pVCpu->hm.s.fForceTLBFlush = true;
1956	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1957	}
1958
1959	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1960	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1961
1962	if (pVCpu->hm.s.fForceTLBFlush)
1963	{
1964	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1965	pVCpu->hm.s.fForceTLBFlush = false;
1966	}
1967	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1968	* where it is commented out. Support individual entry flushing
1969	* someday. */
1970	#if 0
1971	else
1972	{
1973	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1974	{
1975	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1976	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1977	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1978	}
1979	else
1980	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1981
1982	pVCpu->hm.s.TlbShootdown.cPages = 0;
1983	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1984	}
1985	#endif
1986	}
1987
1988
1989	/**
1990	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1991	*
1992	* @returns VBox status code.
1993	* @param pVM Pointer to the VM.
1994	* @param pVCpu Pointer to the VMCPU.
1995	* @param pCpu Pointer to the global HM CPU struct.
1996	*
1997	* @remarks Called with interrupts disabled.
1998	*/
1999	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2000	{
2001	AssertPtr(pVM);
2002	AssertPtr(pVCpu);
2003	AssertPtr(pCpu);
2004	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2005	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2006
2007	/*
2008	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2009	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2010	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2011	*/
2012	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2013	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2014	{
2015	pVCpu->hm.s.fForceTLBFlush = true;
2016	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2017	}
2018
2019	/* Check for explicit TLB shootdown flushes. */
2020	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2021	{
2022	/*
2023	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2024	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2025	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2026	*/
2027	pVCpu->hm.s.fForceTLBFlush = true;
2028	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2029	}
2030
2031	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2032	if (pVCpu->hm.s.fForceTLBFlush)
2033	{
2034	++pCpu->uCurrentAsid;
2035	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2036	{
2037	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2038	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2039	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2040	}
2041
2042	pVCpu->hm.s.fForceTLBFlush = false;
2043	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2044	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2045	if (pCpu->fFlushAsidBeforeUse)
2046	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2047	}
2048	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2049	* where it is commented out. Support individual entry flushing
2050	* someday. */
2051	#if 0
2052	else
2053	{
2054	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2055	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2056	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2057	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2058
2059	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2060	{
2061	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2062	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2063	{
2064	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2065	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2066	}
2067	else
2068	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2069
2070	pVCpu->hm.s.TlbShootdown.cPages = 0;
2071	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2072	}
2073	else
2074	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2075	}
2076	#endif
2077
2078	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2079	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2080	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2081	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2082	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2083	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2084
2085	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2086	AssertRC(rc);
2087	}
2088
2089
2090	/**
2091	* Flushes the guest TLB entry based on CPU capabilities.
2092	*
2093	* @param pVCpu Pointer to the VMCPU.
2094	* @param pCpu Pointer to the global HM CPU struct.
2095	*/
2096	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2097	{
2098	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2099	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2100	#endif
2101	PVM pVM = pVCpu->CTX_SUFF(pVM);
2102	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2103	{
2104	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2105	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2106	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2107	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2108	default:
2109	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2110	break;
2111	}
2112
2113	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2114	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2115
2116	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2117	}
2118
2119
2120	/**
2121	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2122	* TLB entries from the host TLB before VM-entry.
2123	*
2124	* @returns VBox status code.
2125	* @param pVM Pointer to the VM.
2126	*/
2127	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2128	{
2129	/*
2130	* Determine optimal flush type for Nested Paging.
2131	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2132	* guest execution (see hmR3InitFinalizeR0()).
2133	*/
2134	if (pVM->hm.s.fNestedPaging)
2135	{
2136	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2137	{
2138	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2139	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2140	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2142	else
2143	{
2144	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2145	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2146	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2147	}
2148
2149	/* Make sure the write-back cacheable memory type for EPT is supported. */
2150	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2151	{
2152	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2153	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2154	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2155	}
2156	}
2157	else
2158	{
2159	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2160	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2161	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2162	}
2163	}
2164
2165	/*
2166	* Determine optimal flush type for VPID.
2167	*/
2168	if (pVM->hm.s.vmx.fVpid)
2169	{
2170	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2171	{
2172	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2173	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2174	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2175	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2176	else
2177	{
2178	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2179	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2180	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2181	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2182	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2183	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2184	pVM->hm.s.vmx.fVpid = false;
2185	}
2186	}
2187	else
2188	{
2189	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2190	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2191	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2192	pVM->hm.s.vmx.fVpid = false;
2193	}
2194	}
2195
2196	/*
2197	* Setup the handler for flushing tagged-TLBs.
2198	*/
2199	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2200	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2201	else if (pVM->hm.s.fNestedPaging)
2202	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2203	else if (pVM->hm.s.vmx.fVpid)
2204	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2205	else
2206	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2207	return VINF_SUCCESS;
2208	}
2209
2210
2211	/**
2212	* Sets up pin-based VM-execution controls in the VMCS.
2213	*
2214	* @returns VBox status code.
2215	* @param pVM Pointer to the VM.
2216	* @param pVCpu Pointer to the VMCPU.
2217	*/
2218	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2219	{
2220	AssertPtr(pVM);
2221	AssertPtr(pVCpu);
2222
2223	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2224	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2225
2226	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2227	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2228	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2229
2230	/* Enable the VMX preemption timer. */
2231	if (pVM->hm.s.vmx.fUsePreemptTimer)
2232	{
2233	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2234	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2235	}
2236
2237	if ((val & zap) != val)
2238	{
2239	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2240	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2241	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2242	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2243	}
2244
2245	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2246	AssertRCReturn(rc, rc);
2247
2248	/* Update VCPU with the currently set pin-based VM-execution controls. */
2249	pVCpu->hm.s.vmx.u32PinCtls = val;
2250	return rc;
2251	}
2252
2253
2254	/**
2255	* Sets up processor-based VM-execution controls in the VMCS.
2256	*
2257	* @returns VBox status code.
2258	* @param pVM Pointer to the VM.
2259	* @param pVMCPU Pointer to the VMCPU.
2260	*/
2261	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2262	{
2263	AssertPtr(pVM);
2264	AssertPtr(pVCpu);
2265
2266	int rc = VERR_INTERNAL_ERROR_5;
2267	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2268	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2269
2270	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2271	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2272	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2273	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2274	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2275	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2276	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2277
2278	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2279	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2280	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2281	{
2282	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2283	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2284	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2285	}
2286
2287	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2288	if (!pVM->hm.s.fNestedPaging)
2289	{
2290	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2291	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2292	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2293	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2294	}
2295
2296	/* Use TPR shadowing if supported by the CPU. */
2297	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2298	{
2299	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2300	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2301	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2302	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2303	AssertRCReturn(rc, rc);
2304
2305	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2306	/* CR8 writes causes a VM-exit based on TPR threshold. */
2307	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2308	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2309	}
2310	else
2311	{
2312	/*
2313	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2314	* Set this control only for 64-bit guests.
2315	*/
2316	if (pVM->hm.s.fAllow64BitGuests)
2317	{
2318	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2319	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2320	}
2321	}
2322
2323	/* Use MSR-bitmaps if supported by the CPU. */
2324	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2325	{
2326	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2327
2328	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2329	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2330	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2331	AssertRCReturn(rc, rc);
2332
2333	/*
2334	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2335	* automatically as dedicated fields in the VMCS.
2336	*/
2337	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2339	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2342
2343	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2344	/*
2345	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2346	*/
2347	if ( HMVMX_IS_64BIT_HOST_MODE()
2348	&& pVM->hm.s.fAllow64BitGuests)
2349	{
2350	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2351	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2352	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2353	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2354	}
2355	#endif
2356	}
2357
2358	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2359	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2360	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2361
2362	if ((val & zap) != val)
2363	{
2364	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2365	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2366	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2367	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2368	}
2369
2370	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2371	AssertRCReturn(rc, rc);
2372
2373	/* Update VCPU with the currently set processor-based VM-execution controls. */
2374	pVCpu->hm.s.vmx.u32ProcCtls = val;
2375
2376	/*
2377	* Secondary processor-based VM-execution controls.
2378	*/
2379	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2380	{
2381	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2382	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2383
2384	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2385	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2386
2387	if (pVM->hm.s.fNestedPaging)
2388	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2389	else
2390	{
2391	/*
2392	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2393	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2394	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2395	*/
2396	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2397	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2398	}
2399
2400	if (pVM->hm.s.vmx.fVpid)
2401	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2402
2403	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2404	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2405
2406	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2407	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2408	* done dynamically. */
2409	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2410	{
2411	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2412	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2413	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2414	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2415	AssertRCReturn(rc, rc);
2416	}
2417
2418	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2419	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2420
2421	if ((val & zap) != val)
2422	{
2423	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2424	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2425	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2426	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2427	}
2428
2429	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2430	AssertRCReturn(rc, rc);
2431
2432	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2433	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2434	}
2435	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2436	{
2437	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2438	"available\n"));
2439	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2440	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2441	}
2442
2443	return VINF_SUCCESS;
2444	}
2445
2446
2447	/**
2448	* Sets up miscellaneous (everything other than Pin & Processor-based
2449	* VM-execution) control fields in the VMCS.
2450	*
2451	* @returns VBox status code.
2452	* @param pVM Pointer to the VM.
2453	* @param pVCpu Pointer to the VMCPU.
2454	*/
2455	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2456	{
2457	NOREF(pVM);
2458	AssertPtr(pVM);
2459	AssertPtr(pVCpu);
2460
2461	int rc = VERR_GENERAL_FAILURE;
2462
2463	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2464	#if 0
2465	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2466	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2467	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2468
2469	/*
2470	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2471	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2472	* We thus use the exception bitmap to control it rather than use both.
2473	*/
2474	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2476
2477	/** @todo Explore possibility of using IO-bitmaps. */
2478	/* All IO & IOIO instructions cause VM-exits. */
2479	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2480	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2481
2482	/* Initialize the MSR-bitmap area. */
2483	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2484	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2485	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2486	#endif
2487
2488	/* Setup MSR auto-load/store area. */
2489	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2490	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2491	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2492	AssertRCReturn(rc, rc);
2493	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2494	AssertRCReturn(rc, rc);
2495
2496	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2497	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2498	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2499	AssertRCReturn(rc, rc);
2500
2501	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2502	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2503	AssertRCReturn(rc, rc);
2504
2505	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2506	#if 0
2507	/* Setup debug controls */
2508	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2509	AssertRCReturn(rc, rc);
2510	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2511	AssertRCReturn(rc, rc);
2512	#endif
2513
2514	return rc;
2515	}
2516
2517
2518	/**
2519	* Sets up the initial exception bitmap in the VMCS based on static conditions
2520	* (i.e. conditions that cannot ever change after starting the VM).
2521	*
2522	* @returns VBox status code.
2523	* @param pVM Pointer to the VM.
2524	* @param pVCpu Pointer to the VMCPU.
2525	*/
2526	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2527	{
2528	AssertPtr(pVM);
2529	AssertPtr(pVCpu);
2530
2531	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2532
2533	uint32_t u32XcptBitmap = 0;
2534
2535	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2536	if (!pVM->hm.s.fNestedPaging)
2537	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2538
2539	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2540	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2541	AssertRCReturn(rc, rc);
2542	return rc;
2543	}
2544
2545
2546	/**
2547	* Sets up the initial guest-state mask. The guest-state mask is consulted
2548	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2549	* for the nested virtualization case (as it would cause a VM-exit).
2550	*
2551	* @param pVCpu Pointer to the VMCPU.
2552	*/
2553	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2554	{
2555	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2556	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2557	return VINF_SUCCESS;
2558	}
2559
2560
2561	/**
2562	* Does per-VM VT-x initialization.
2563	*
2564	* @returns VBox status code.
2565	* @param pVM Pointer to the VM.
2566	*/
2567	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2568	{
2569	LogFlowFunc(("pVM=%p\n", pVM));
2570
2571	int rc = hmR0VmxStructsAlloc(pVM);
2572	if (RT_FAILURE(rc))
2573	{
2574	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2575	return rc;
2576	}
2577
2578	return VINF_SUCCESS;
2579	}
2580
2581
2582	/**
2583	* Does per-VM VT-x termination.
2584	*
2585	* @returns VBox status code.
2586	* @param pVM Pointer to the VM.
2587	*/
2588	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2589	{
2590	LogFlowFunc(("pVM=%p\n", pVM));
2591
2592	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2593	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2594	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2595	#endif
2596	hmR0VmxStructsFree(pVM);
2597	return VINF_SUCCESS;
2598	}
2599
2600
2601	/**
2602	* Sets up the VM for execution under VT-x.
2603	* This function is only called once per-VM during initialization.
2604	*
2605	* @returns VBox status code.
2606	* @param pVM Pointer to the VM.
2607	*/
2608	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2609	{
2610	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2611	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2612
2613	LogFlowFunc(("pVM=%p\n", pVM));
2614
2615	/*
2616	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2617	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2618	*/
2619	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2620	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2621	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2622	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2623	{
2624	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2625	return VERR_INTERNAL_ERROR;
2626	}
2627
2628	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2629	/*
2630	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2631	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2632	*/
2633	if ( pVM->hm.s.fAllow64BitGuests
2634	&& !HMVMX_IS_64BIT_HOST_MODE())
2635	{
2636	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2637	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2638	}
2639	#endif
2640
2641	/* Initialize these always, see hmR3InitFinalizeR0().*/
2642	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2643	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2644
2645	/* Setup the tagged-TLB flush handlers. */
2646	int rc = hmR0VmxSetupTaggedTlb(pVM);
2647	if (RT_FAILURE(rc))
2648	{
2649	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2650	return rc;
2651	}
2652
2653	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2654	{
2655	PVMCPU pVCpu = &pVM->aCpus[i];
2656	AssertPtr(pVCpu);
2657	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2658
2659	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2660	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2661
2662	/* Set revision dword at the beginning of the VMCS structure. */
2663	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2664
2665	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2666	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2667	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2668	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2669
2670	/* Load this VMCS as the current VMCS. */
2671	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2672	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2673	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2674
2675	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2676	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2677	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2678
2679	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2680	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2681	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2682
2683	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2684	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2685	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2686
2687	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2688	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2689	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2690
2691	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2692	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2693	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2694
2695	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2696	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2697	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2698	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2699	#endif
2700
2701	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2702	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2703	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2704	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2705
2706	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2707
2708	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2709	}
2710
2711	return VINF_SUCCESS;
2712	}
2713
2714
2715	/**
2716	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2717	* the VMCS.
2718	*
2719	* @returns VBox status code.
2720	* @param pVM Pointer to the VM.
2721	* @param pVCpu Pointer to the VMCPU.
2722	*/
2723	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2724	{
2725	NOREF(pVM); NOREF(pVCpu);
2726
2727	RTCCUINTREG uReg = ASMGetCR0();
2728	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2729	AssertRCReturn(rc, rc);
2730
2731	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2732	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2733	if (HMVMX_IS_64BIT_HOST_MODE())
2734	{
2735	uint64_t uRegCR3 = HMR0Get64bitCR3();
2736	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2737	}
2738	else
2739	#endif
2740	{
2741	uReg = ASMGetCR3();
2742	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2743	}
2744	AssertRCReturn(rc, rc);
2745
2746	uReg = ASMGetCR4();
2747	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2748	AssertRCReturn(rc, rc);
2749	return rc;
2750	}
2751
2752
2753	#if HC_ARCH_BITS == 64
2754	/**
2755	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2756	* requirements. See hmR0VmxSaveHostSegmentRegs().
2757	*/
2758	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2759	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2760	{ \
2761	bool fValidSelector = true; \
2762	if ((selValue) & X86_SEL_LDT) \
2763	{ \
2764	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2765	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2766	} \
2767	if (fValidSelector) \
2768	{ \
2769	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2770	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2771	} \
2772	(selValue) = 0; \
2773	}
2774	#endif
2775
2776
2777	/**
2778	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2779	* the host-state area in the VMCS.
2780	*
2781	* @returns VBox status code.
2782	* @param pVM Pointer to the VM.
2783	* @param pVCpu Pointer to the VMCPU.
2784	*/
2785	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2786	{
2787	NOREF(pVM);
2788	int rc = VERR_INTERNAL_ERROR_5;
2789
2790	/*
2791	* Host DS, ES, FS and GS segment registers.
2792	*/
2793	#if HC_ARCH_BITS == 64
2794	RTSEL uSelDS = ASMGetDS();
2795	RTSEL uSelES = ASMGetES();
2796	RTSEL uSelFS = ASMGetFS();
2797	RTSEL uSelGS = ASMGetGS();
2798	#else
2799	RTSEL uSelDS = 0;
2800	RTSEL uSelES = 0;
2801	RTSEL uSelFS = 0;
2802	RTSEL uSelGS = 0;
2803	#endif
2804
2805	/* Recalculate which host-state bits need to be manually restored. */
2806	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2807
2808	/*
2809	* Host CS and SS segment registers.
2810	*/
2811	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2812	RTSEL uSelCS;
2813	RTSEL uSelSS;
2814	if (HMVMX_IS_64BIT_HOST_MODE())
2815	{
2816	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2817	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2818	}
2819	else
2820	{
2821	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2822	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2823	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2824	}
2825	#else
2826	RTSEL uSelCS = ASMGetCS();
2827	RTSEL uSelSS = ASMGetSS();
2828	#endif
2829
2830	/*
2831	* Host TR segment register.
2832	*/
2833	RTSEL uSelTR = ASMGetTR();
2834
2835	#if HC_ARCH_BITS == 64
2836	/*
2837	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2838	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2839	*/
2840	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2841	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2842	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2843	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2844	# undef VMXLOCAL_ADJUST_HOST_SEG
2845	#endif
2846
2847	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2848	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2849	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2850	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2851	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2852	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2853	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2854	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2855	Assert(uSelCS);
2856	Assert(uSelTR);
2857
2858	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2859	#if 0
2860	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2861	Assert(uSelSS != 0);
2862	#endif
2863
2864	/* Write these host selector fields into the host-state area in the VMCS. */
2865	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2866	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2867	#if HC_ARCH_BITS == 64
2868	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2869	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2870	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2871	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2872	#endif
2873	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2874
2875	/*
2876	* Host GDTR and IDTR.
2877	*/
2878	RTGDTR Gdtr;
2879	RT_ZERO(Gdtr);
2880	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2881	if (HMVMX_IS_64BIT_HOST_MODE())
2882	{
2883	X86XDTR64 Gdtr64;
2884	X86XDTR64 Idtr64;
2885	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2886	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2887	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2888
2889	Gdtr.cbGdt = Gdtr64.cb;
2890	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2891	}
2892	else
2893	#endif
2894	{
2895	RTIDTR Idtr;
2896	ASMGetGDTR(&Gdtr);
2897	ASMGetIDTR(&Idtr);
2898	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2899	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2900
2901	#if HC_ARCH_BITS == 64
2902	/*
2903	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2904	* maximum limit (0xffff) on every VM-exit.
2905	*/
2906	if (Gdtr.cbGdt != 0xffff)
2907	{
2908	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2909	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2910	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2911	}
2912
2913	/*
2914	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2915	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2916	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2917	*/
2918	if (Idtr.cbIdt < 0x0fff)
2919	{
2920	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2921	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2922	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2923	}
2924	#endif
2925	}
2926
2927	/*
2928	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2929	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2930	*/
2931	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2932	{
2933	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2934	return VERR_VMX_INVALID_HOST_STATE;
2935	}
2936
2937	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2938	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2939	if (HMVMX_IS_64BIT_HOST_MODE())
2940	{
2941	/* We need the 64-bit TR base for hybrid darwin. */
2942	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2943	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2944	}
2945	else
2946	#endif
2947	{
2948	uintptr_t uTRBase;
2949	#if HC_ARCH_BITS == 64
2950	uTRBase = X86DESC64_BASE(pDesc);
2951
2952	/*
2953	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2954	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2955	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2956	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2957	*
2958	* [1] See Intel spec. 3.5 "System Descriptor Types".
2959	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2960	*/
2961	Assert(pDesc->System.u4Type == 11);
2962	if ( pDesc->System.u16LimitLow != 0x67
2963	\|\| pDesc->System.u4LimitHigh)
2964	{
2965	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2966	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2967
2968	/* Store the GDTR here as we need it while restoring TR. */
2969	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2970	}
2971	#else
2972	uTRBase = X86DESC_BASE(pDesc);
2973	#endif
2974	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2975	}
2976	AssertRCReturn(rc, rc);
2977
2978	/*
2979	* Host FS base and GS base.
2980	*/
2981	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2982	if (HMVMX_IS_64BIT_HOST_MODE())
2983	{
2984	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2985	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2986	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2987	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2988
2989	# if HC_ARCH_BITS == 64
2990	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2991	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2992	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2993	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2994	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2995	# endif
2996	}
2997	#endif
2998	return rc;
2999	}
3000
3001
3002	/**
3003	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3004	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3005	* the host after every successful VM exit.
3006	*
3007	* @returns VBox status code.
3008	* @param pVM Pointer to the VM.
3009	* @param pVCpu Pointer to the VMCPU.
3010	*
3011	* @remarks No-long-jump zone!!!
3012	*/
3013	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3014	{
3015	NOREF(pVM);
3016
3017	AssertPtr(pVCpu);
3018	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3019
3020	int rc = VINF_SUCCESS;
3021	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3022	if ( HMVMX_IS_64BIT_HOST_MODE()
3023	&& pVM->hm.s.fAllow64BitGuests)
3024	{
3025	hmR0VmxLazySaveHostMsrs(pVCpu);
3026	}
3027	#endif
3028
3029	if (pVCpu->hm.s.vmx.cMsrs > 0)
3030	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3031
3032	/*
3033	* Host Sysenter MSRs.
3034	*/
3035	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3036	AssertRCReturn(rc, rc);
3037	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3038	if (HMVMX_IS_64BIT_HOST_MODE())
3039	{
3040	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3041	AssertRCReturn(rc, rc);
3042	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3043	}
3044	else
3045	{
3046	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3047	AssertRCReturn(rc, rc);
3048	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3049	}
3050	#elif HC_ARCH_BITS == 32
3051	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3052	AssertRCReturn(rc, rc);
3053	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3054	#else
3055	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3056	AssertRCReturn(rc, rc);
3057	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3058	#endif
3059	AssertRCReturn(rc, rc);
3060
3061	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3062	* hmR0VmxSetupExitCtls() !! */
3063	return rc;
3064	}
3065
3066
3067	/**
3068	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3069	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3070	* controls".
3071	*
3072	* @returns VBox status code.
3073	* @param pVCpu Pointer to the VMCPU.
3074	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3075	* out-of-sync. Make sure to update the required fields
3076	* before using them.
3077	*
3078	* @remarks No-long-jump zone!!!
3079	*/
3080	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3081	{
3082	int rc = VINF_SUCCESS;
3083	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3084	{
3085	PVM pVM = pVCpu->CTX_SUFF(pVM);
3086	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3087	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3088
3089	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3090	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3091
3092	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3093	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3094	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3095	else
3096	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3097
3098	/*
3099	* The following should -not- be set (since we're not in SMM mode):
3100	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3101	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3102	*/
3103
3104	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3105	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3106	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3107
3108	if ((val & zap) != val)
3109	{
3110	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3111	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3112	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3113	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3114	}
3115
3116	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3117	AssertRCReturn(rc, rc);
3118
3119	/* Update VCPU with the currently set VM-exit controls. */
3120	pVCpu->hm.s.vmx.u32EntryCtls = val;
3121	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3122	}
3123	return rc;
3124	}
3125
3126
3127	/**
3128	* Sets up the VM-exit controls in the VMCS.
3129	*
3130	* @returns VBox status code.
3131	* @param pVM Pointer to the VM.
3132	* @param pVCpu Pointer to the VMCPU.
3133	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3134	* out-of-sync. Make sure to update the required fields
3135	* before using them.
3136	*
3137	* @remarks requires EFER.
3138	*/
3139	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3140	{
3141	NOREF(pMixedCtx);
3142
3143	int rc = VINF_SUCCESS;
3144	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3145	{
3146	PVM pVM = pVCpu->CTX_SUFF(pVM);
3147	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3148	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3149
3150	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3151	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3152
3153	/*
3154	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3155	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3156	*/
3157	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3158	if (HMVMX_IS_64BIT_HOST_MODE())
3159	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3160	else
3161	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3162	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3163	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3164	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3165	else
3166	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3167	#endif
3168
3169	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3170	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3171
3172	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3173	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3174	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3175	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3176	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3177
3178	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3179	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3180
3181	if ((val & zap) != val)
3182	{
3183	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3184	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3185	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3186	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3187	}
3188
3189	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3190	AssertRCReturn(rc, rc);
3191
3192	/* Update VCPU with the currently set VM-exit controls. */
3193	pVCpu->hm.s.vmx.u32ExitCtls = val;
3194	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3195	}
3196	return rc;
3197	}
3198
3199
3200	/**
3201	* Loads the guest APIC and related state.
3202	*
3203	* @returns VBox status code.
3204	* @param pVM Pointer to the VM.
3205	* @param pVCpu Pointer to the VMCPU.
3206	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3207	* out-of-sync. Make sure to update the required fields
3208	* before using them.
3209	*/
3210	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3211	{
3212	NOREF(pMixedCtx);
3213
3214	int rc = VINF_SUCCESS;
3215	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3216	{
3217	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3218	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3219	{
3220	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3221
3222	bool fPendingIntr = false;
3223	uint8_t u8Tpr = 0;
3224	uint8_t u8PendingIntr = 0;
3225	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3226	AssertRCReturn(rc, rc);
3227
3228	/*
3229	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3230	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3231	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3232	* the interrupt when we VM-exit for other reasons.
3233	*/
3234	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3235	uint32_t u32TprThreshold = 0;
3236	if (fPendingIntr)
3237	{
3238	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3239	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3240	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3241	if (u8PendingPriority <= u8TprPriority)
3242	u32TprThreshold = u8PendingPriority;
3243	else
3244	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3245	}
3246	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3247
3248	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3249	AssertRCReturn(rc, rc);
3250	}
3251
3252	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3253	}
3254	return rc;
3255	}
3256
3257
3258	/**
3259	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3260	*
3261	* @returns Guest's interruptibility-state.
3262	* @param pVCpu Pointer to the VMCPU.
3263	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3264	* out-of-sync. Make sure to update the required fields
3265	* before using them.
3266	*
3267	* @remarks No-long-jump zone!!!
3268	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3269	*/
3270	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3271	{
3272	/*
3273	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3274	* inhibit interrupts or clear any existing interrupt-inhibition.
3275	*/
3276	uint32_t uIntrState = 0;
3277	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3278	{
3279	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3280	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
3281	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3282	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3283	{
3284	/*
3285	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3286	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3287	*/
3288	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3289	}
3290	else if (pMixedCtx->eflags.Bits.u1IF)
3291	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3292	else
3293	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3294	}
3295	return uIntrState;
3296	}
3297
3298
3299	/**
3300	* Loads the guest's interruptibility-state into the guest-state area in the
3301	* VMCS.
3302	*
3303	* @returns VBox status code.
3304	* @param pVCpu Pointer to the VMCPU.
3305	* @param uIntrState The interruptibility-state to set.
3306	*/
3307	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3308	{
3309	NOREF(pVCpu);
3310	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3311	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3312	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3313	AssertRCReturn(rc, rc);
3314	return rc;
3315	}
3316
3317
3318	/**
3319	* Loads the guest's RIP into the guest-state area in the VMCS.
3320	*
3321	* @returns VBox status code.
3322	* @param pVCpu Pointer to the VMCPU.
3323	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3324	* out-of-sync. Make sure to update the required fields
3325	* before using them.
3326	*
3327	* @remarks No-long-jump zone!!!
3328	*/
3329	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3330	{
3331	int rc = VINF_SUCCESS;
3332	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3333	{
3334	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3335	AssertRCReturn(rc, rc);
3336
3337	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3338	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3339	}
3340	return rc;
3341	}
3342
3343
3344	/**
3345	* Loads the guest's RSP into the guest-state area in the VMCS.
3346	*
3347	* @returns VBox status code.
3348	* @param pVCpu Pointer to the VMCPU.
3349	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3350	* out-of-sync. Make sure to update the required fields
3351	* before using them.
3352	*
3353	* @remarks No-long-jump zone!!!
3354	*/
3355	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3356	{
3357	int rc = VINF_SUCCESS;
3358	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3359	{
3360	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3361	AssertRCReturn(rc, rc);
3362
3363	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3364	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3365	}
3366	return rc;
3367	}
3368
3369
3370	/**
3371	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3372	*
3373	* @returns VBox status code.
3374	* @param pVCpu Pointer to the VMCPU.
3375	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3376	* out-of-sync. Make sure to update the required fields
3377	* before using them.
3378	*
3379	* @remarks No-long-jump zone!!!
3380	*/
3381	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3382	{
3383	int rc = VINF_SUCCESS;
3384	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3385	{
3386	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3387	Let us assert it as such and use 32-bit VMWRITE. */
3388	Assert(!(pMixedCtx->rflags.u64 >> 32));
3389	X86EFLAGS Eflags = pMixedCtx->eflags;
3390	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3391	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3392
3393	/*
3394	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3395	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3396	*/
3397	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3398	{
3399	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3400	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3401	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3402	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3403	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3404	}
3405
3406	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3407	AssertRCReturn(rc, rc);
3408
3409	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3410	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3411	}
3412	return rc;
3413	}
3414
3415
3416	/**
3417	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3418	*
3419	* @returns VBox status code.
3420	* @param pVCpu Pointer to the VMCPU.
3421	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3422	* out-of-sync. Make sure to update the required fields
3423	* before using them.
3424	*
3425	* @remarks No-long-jump zone!!!
3426	*/
3427	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3428	{
3429	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3430	AssertRCReturn(rc, rc);
3431	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3432	AssertRCReturn(rc, rc);
3433	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3434	AssertRCReturn(rc, rc);
3435	return rc;
3436	}
3437
3438
3439	/**
3440	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3441	* CR0 is partially shared with the host and we have to consider the FPU bits.
3442	*
3443	* @returns VBox status code.
3444	* @param pVM Pointer to the VM.
3445	* @param pVCpu Pointer to the VMCPU.
3446	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3447	* out-of-sync. Make sure to update the required fields
3448	* before using them.
3449	*
3450	* @remarks No-long-jump zone!!!
3451	*/
3452	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3453	{
3454	/*
3455	* Guest CR0.
3456	* Guest FPU.
3457	*/
3458	int rc = VINF_SUCCESS;
3459	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3460	{
3461	Assert(!(pMixedCtx->cr0 >> 32));
3462	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3463	PVM pVM = pVCpu->CTX_SUFF(pVM);
3464
3465	/* The guest's view (read access) of its CR0 is unblemished. */
3466	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3467	AssertRCReturn(rc, rc);
3468	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3469
3470	/* Setup VT-x's view of the guest CR0. */
3471	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3472	if (pVM->hm.s.fNestedPaging)
3473	{
3474	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3475	{
3476	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3477	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3478	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3479	}
3480	else
3481	{
3482	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3483	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3484	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3485	}
3486
3487	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3488	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3489	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3490
3491	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3492	AssertRCReturn(rc, rc);
3493	}
3494	else
3495	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3496
3497	/*
3498	* Guest FPU bits.
3499	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3500	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3501	*/
3502	u32GuestCR0 \|= X86_CR0_NE;
3503	bool fInterceptNM = false;
3504	if (CPUMIsGuestFPUStateActive(pVCpu))
3505	{
3506	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3507	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3508	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3509	}
3510	else
3511	{
3512	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3513	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3514	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3515	}
3516
3517	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3518	bool fInterceptMF = false;
3519	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3520	fInterceptMF = true;
3521
3522	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3523	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3524	{
3525	Assert(PDMVmmDevHeapIsEnabled(pVM));
3526	Assert(pVM->hm.s.vmx.pRealModeTSS);
3527	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3528	fInterceptNM = true;
3529	fInterceptMF = true;
3530	}
3531	else
3532	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3533
3534	if (fInterceptNM)
3535	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3536	else
3537	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3538
3539	if (fInterceptMF)
3540	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3541	else
3542	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3543
3544	/* Additional intercepts for debugging, define these yourself explicitly. */
3545	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3546	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3547	\| RT_BIT(X86_XCPT_BP)
3548	\| RT_BIT(X86_XCPT_DB)
3549	\| RT_BIT(X86_XCPT_DE)
3550	\| RT_BIT(X86_XCPT_NM)
3551	\| RT_BIT(X86_XCPT_UD)
3552	\| RT_BIT(X86_XCPT_NP)
3553	\| RT_BIT(X86_XCPT_SS)
3554	\| RT_BIT(X86_XCPT_GP)
3555	\| RT_BIT(X86_XCPT_PF)
3556	\| RT_BIT(X86_XCPT_MF)
3557	;
3558	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3559	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3560	#endif
3561
3562	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3563
3564	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3565	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3566	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3567	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3568	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3569	else
3570	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3571
3572	u32GuestCR0 \|= uSetCR0;
3573	u32GuestCR0 &= uZapCR0;
3574	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3575
3576	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3577	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3578	AssertRCReturn(rc, rc);
3579	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3580	AssertRCReturn(rc, rc);
3581	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3582
3583	/*
3584	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3585	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3586	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3587	*/
3588	uint32_t u32CR0Mask = 0;
3589	u32CR0Mask = X86_CR0_PE
3590	\| X86_CR0_NE
3591	\| X86_CR0_WP
3592	\| X86_CR0_PG
3593	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3594	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3595	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3596
3597	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3598	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3599	* and @bugref{6944}. */
3600	#if 0
3601	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3602	u32CR0Mask &= ~X86_CR0_PE;
3603	#endif
3604	if (pVM->hm.s.fNestedPaging)
3605	u32CR0Mask &= ~X86_CR0_WP;
3606
3607	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3608	if (fInterceptNM)
3609	{
3610	u32CR0Mask \|= X86_CR0_TS
3611	\| X86_CR0_MP;
3612	}
3613
3614	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3615	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3616	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3617	AssertRCReturn(rc, rc);
3618	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3619
3620	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3621	}
3622	return rc;
3623	}
3624
3625
3626	/**
3627	* Loads the guest control registers (CR3, CR4) into the guest-state area
3628	* in the VMCS.
3629	*
3630	* @returns VBox status code.
3631	* @param pVM Pointer to the VM.
3632	* @param pVCpu Pointer to the VMCPU.
3633	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3634	* out-of-sync. Make sure to update the required fields
3635	* before using them.
3636	*
3637	* @remarks No-long-jump zone!!!
3638	*/
3639	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3640	{
3641	int rc = VINF_SUCCESS;
3642	PVM pVM = pVCpu->CTX_SUFF(pVM);
3643
3644	/*
3645	* Guest CR2.
3646	* It's always loaded in the assembler code. Nothing to do here.
3647	*/
3648
3649	/*
3650	* Guest CR3.
3651	*/
3652	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3653	{
3654	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3655	if (pVM->hm.s.fNestedPaging)
3656	{
3657	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3658
3659	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3660	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3661	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3662	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3663
3664	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3665	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3666	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3667
3668	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3669	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3670	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3671	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3672
3673	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3674	AssertRCReturn(rc, rc);
3675	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3676
3677	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3678	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3679	{
3680	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3681	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3682	{
3683	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3684	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3685	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3686	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3687	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3688	}
3689
3690	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3691	have Unrestricted Execution to handle the guest when it's not using paging. */
3692	GCPhysGuestCR3 = pMixedCtx->cr3;
3693	}
3694	else
3695	{
3696	/*
3697	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3698	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3699	* EPT takes care of translating it to host-physical addresses.
3700	*/
3701	RTGCPHYS GCPhys;
3702	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3703	Assert(PDMVmmDevHeapIsEnabled(pVM));
3704
3705	/* We obtain it here every time as the guest could have relocated this PCI region. */
3706	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3707	AssertRCReturn(rc, rc);
3708
3709	GCPhysGuestCR3 = GCPhys;
3710	}
3711
3712	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3713	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3714	}
3715	else
3716	{
3717	/* Non-nested paging case, just use the hypervisor's CR3. */
3718	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3719
3720	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3721	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3722	}
3723	AssertRCReturn(rc, rc);
3724
3725	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3726	}
3727
3728	/*
3729	* Guest CR4.
3730	*/
3731	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3732	{
3733	Assert(!(pMixedCtx->cr4 >> 32));
3734	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3735
3736	/* The guest's view of its CR4 is unblemished. */
3737	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3738	AssertRCReturn(rc, rc);
3739	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3740
3741	/* Setup VT-x's view of the guest CR4. */
3742	/*
3743	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3744	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3745	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3746	*/
3747	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3748	{
3749	Assert(pVM->hm.s.vmx.pRealModeTSS);
3750	Assert(PDMVmmDevHeapIsEnabled(pVM));
3751	u32GuestCR4 &= ~X86_CR4_VME;
3752	}
3753
3754	if (pVM->hm.s.fNestedPaging)
3755	{
3756	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3757	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3758	{
3759	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3760	u32GuestCR4 \|= X86_CR4_PSE;
3761	/* Our identity mapping is a 32-bit page directory. */
3762	u32GuestCR4 &= ~X86_CR4_PAE;
3763	}
3764	/* else use guest CR4.*/
3765	}
3766	else
3767	{
3768	/*
3769	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3770	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3771	*/
3772	switch (pVCpu->hm.s.enmShadowMode)
3773	{
3774	case PGMMODE_REAL: /* Real-mode. */
3775	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3776	case PGMMODE_32_BIT: /* 32-bit paging. */
3777	{
3778	u32GuestCR4 &= ~X86_CR4_PAE;
3779	break;
3780	}
3781
3782	case PGMMODE_PAE: /* PAE paging. */
3783	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3784	{
3785	u32GuestCR4 \|= X86_CR4_PAE;
3786	break;
3787	}
3788
3789	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3790	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3791	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3792	break;
3793	#endif
3794	default:
3795	AssertFailed();
3796	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3797	}
3798	}
3799
3800	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3801	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3802	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3803	u32GuestCR4 \|= uSetCR4;
3804	u32GuestCR4 &= uZapCR4;
3805
3806	/* Write VT-x's view of the guest CR4 into the VMCS. */
3807	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3808	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3809	AssertRCReturn(rc, rc);
3810
3811	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3812	uint32_t u32CR4Mask = 0;
3813	u32CR4Mask = X86_CR4_VME
3814	\| X86_CR4_PAE
3815	\| X86_CR4_PGE
3816	\| X86_CR4_PSE
3817	\| X86_CR4_VMXE;
3818	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3819	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3820	AssertRCReturn(rc, rc);
3821
3822	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3823	}
3824	return rc;
3825	}
3826
3827
3828	/**
3829	* Loads the guest debug registers into the guest-state area in the VMCS.
3830	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3831	*
3832	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3833	*
3834	* @returns VBox status code.
3835	* @param pVCpu Pointer to the VMCPU.
3836	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3837	* out-of-sync. Make sure to update the required fields
3838	* before using them.
3839	*
3840	* @remarks No-long-jump zone!!!
3841	*/
3842	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3843	{
3844	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3845	return VINF_SUCCESS;
3846
3847	#ifdef VBOX_STRICT
3848	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3849	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3850	{
3851	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3852	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3853	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3854	}
3855	#endif
3856
3857	int rc;
3858	PVM pVM = pVCpu->CTX_SUFF(pVM);
3859	bool fInterceptDB = false;
3860	bool fInterceptMovDRx = false;
3861	if ( pVCpu->hm.s.fSingleInstruction
3862	\|\| DBGFIsStepping(pVCpu))
3863	{
3864	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3865	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3866	{
3867	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3868	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3869	AssertRCReturn(rc, rc);
3870	Assert(fInterceptDB == false);
3871	}
3872	else
3873	{
3874	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3875	pVCpu->hm.s.fClearTrapFlag = true;
3876	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3877	fInterceptDB = true;
3878	}
3879	}
3880
3881	if ( fInterceptDB
3882	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3883	{
3884	/*
3885	* Use the combined guest and host DRx values found in the hypervisor
3886	* register set because the debugger has breakpoints active or someone
3887	* is single stepping on the host side without a monitor trap flag.
3888	*
3889	* Note! DBGF expects a clean DR6 state before executing guest code.
3890	*/
3891	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3892	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3893	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3894	{
3895	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3896	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3897	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3898	}
3899	else
3900	#endif
3901	if (!CPUMIsHyperDebugStateActive(pVCpu))
3902	{
3903	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3904	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3905	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3906	}
3907
3908	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3909	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3910	AssertRCReturn(rc, rc);
3911
3912	pVCpu->hm.s.fUsingHyperDR7 = true;
3913	fInterceptDB = true;
3914	fInterceptMovDRx = true;
3915	}
3916	else
3917	{
3918	/*
3919	* If the guest has enabled debug registers, we need to load them prior to
3920	* executing guest code so they'll trigger at the right time.
3921	*/
3922	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3923	{
3924	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3925	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3926	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3927	{
3928	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3929	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3930	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3931	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3932	}
3933	else
3934	#endif
3935	if (!CPUMIsGuestDebugStateActive(pVCpu))
3936	{
3937	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3938	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3939	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3940	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3941	}
3942	Assert(!fInterceptDB);
3943	Assert(!fInterceptMovDRx);
3944	}
3945	/*
3946	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3947	* must intercept #DB in order to maintain a correct DR6 guest value.
3948	*/
3949	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3950	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3951	&& !CPUMIsGuestDebugStateActive(pVCpu))
3952	#else
3953	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3954	#endif
3955	{
3956	fInterceptMovDRx = true;
3957	fInterceptDB = true;
3958	}
3959
3960	/* Update guest DR7. */
3961	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3962	AssertRCReturn(rc, rc);
3963
3964	pVCpu->hm.s.fUsingHyperDR7 = false;
3965	}
3966
3967	/*
3968	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3969	*/
3970	if (fInterceptDB)
3971	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3972	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3973	{
3974	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3975	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3976	#endif
3977	}
3978	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3979	AssertRCReturn(rc, rc);
3980
3981	/*
3982	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3983	*/
3984	if (fInterceptMovDRx)
3985	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3986	else
3987	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3988	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3989	AssertRCReturn(rc, rc);
3990
3991	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3992	return VINF_SUCCESS;
3993	}
3994
3995
3996	#ifdef VBOX_STRICT
3997	/**
3998	* Strict function to validate segment registers.
3999	*
4000	* @remarks ASSUMES CR0 is up to date.
4001	*/
4002	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4003	{
4004	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4005	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4006	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4007	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4008	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4009	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4010	{
4011	/* Protected mode checks */
4012	/* CS */
4013	Assert(pCtx->cs.Attr.n.u1Present);
4014	Assert(!(pCtx->cs.Attr.u & 0xf00));
4015	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4016	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4017	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4018	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4019	\|\| (pCtx->cs.Attr.n.u1Granularity));
4020	/* CS cannot be loaded with NULL in protected mode. */
4021	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4022	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4023	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4024	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4025	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4026	else
4027	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4028	/* SS */
4029	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4030	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4031	if ( !(pCtx->cr0 & X86_CR0_PE)
4032	\|\| pCtx->cs.Attr.n.u4Type == 3)
4033	{
4034	Assert(!pCtx->ss.Attr.n.u2Dpl);
4035	}
4036	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4037	{
4038	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4039	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4040	Assert(pCtx->ss.Attr.n.u1Present);
4041	Assert(!(pCtx->ss.Attr.u & 0xf00));
4042	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4043	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4044	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4045	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4046	\|\| (pCtx->ss.Attr.n.u1Granularity));
4047	}
4048	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4049	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4050	{
4051	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4052	Assert(pCtx->ds.Attr.n.u1Present);
4053	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4054	Assert(!(pCtx->ds.Attr.u & 0xf00));
4055	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4056	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4057	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4058	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4059	\|\| (pCtx->ds.Attr.n.u1Granularity));
4060	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4061	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4062	}
4063	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4064	{
4065	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4066	Assert(pCtx->es.Attr.n.u1Present);
4067	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4068	Assert(!(pCtx->es.Attr.u & 0xf00));
4069	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4070	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4071	\|\| !(pCtx->es.Attr.n.u1Granularity));
4072	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4073	\|\| (pCtx->es.Attr.n.u1Granularity));
4074	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4075	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4076	}
4077	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4078	{
4079	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4080	Assert(pCtx->fs.Attr.n.u1Present);
4081	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4082	Assert(!(pCtx->fs.Attr.u & 0xf00));
4083	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4084	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4085	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4086	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4087	\|\| (pCtx->fs.Attr.n.u1Granularity));
4088	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4089	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4090	}
4091	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4092	{
4093	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4094	Assert(pCtx->gs.Attr.n.u1Present);
4095	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4096	Assert(!(pCtx->gs.Attr.u & 0xf00));
4097	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4098	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4099	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4100	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4101	\|\| (pCtx->gs.Attr.n.u1Granularity));
4102	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4103	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4104	}
4105	/* 64-bit capable CPUs. */
4106	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4107	Assert(!(pCtx->cs.u64Base >> 32));
4108	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4109	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4110	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4111	# endif
4112	}
4113	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4114	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4115	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4116	{
4117	/* Real and v86 mode checks. */
4118	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4119	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4120	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4121	{
4122	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4123	}
4124	else
4125	{
4126	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4127	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4128	}
4129
4130	/* CS */
4131	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4132	Assert(pCtx->cs.u32Limit == 0xffff);
4133	Assert(u32CSAttr == 0xf3);
4134	/* SS */
4135	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4136	Assert(pCtx->ss.u32Limit == 0xffff);
4137	Assert(u32SSAttr == 0xf3);
4138	/* DS */
4139	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4140	Assert(pCtx->ds.u32Limit == 0xffff);
4141	Assert(u32DSAttr == 0xf3);
4142	/* ES */
4143	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4144	Assert(pCtx->es.u32Limit == 0xffff);
4145	Assert(u32ESAttr == 0xf3);
4146	/* FS */
4147	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4148	Assert(pCtx->fs.u32Limit == 0xffff);
4149	Assert(u32FSAttr == 0xf3);
4150	/* GS */
4151	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4152	Assert(pCtx->gs.u32Limit == 0xffff);
4153	Assert(u32GSAttr == 0xf3);
4154	/* 64-bit capable CPUs. */
4155	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4156	Assert(!(pCtx->cs.u64Base >> 32));
4157	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4158	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4159	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4160	# endif
4161	}
4162	}
4163	#endif /* VBOX_STRICT */
4164
4165
4166	/**
4167	* Writes a guest segment register into the guest-state area in the VMCS.
4168	*
4169	* @returns VBox status code.
4170	* @param pVCpu Pointer to the VMCPU.
4171	* @param idxSel Index of the selector in the VMCS.
4172	* @param idxLimit Index of the segment limit in the VMCS.
4173	* @param idxBase Index of the segment base in the VMCS.
4174	* @param idxAccess Index of the access rights of the segment in the VMCS.
4175	* @param pSelReg Pointer to the segment selector.
4176	*
4177	* @remarks No-long-jump zone!!!
4178	*/
4179	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4180	uint32_t idxAccess, PCPUMSELREG pSelReg)
4181	{
4182	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4183	AssertRCReturn(rc, rc);
4184	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4185	AssertRCReturn(rc, rc);
4186	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4187	AssertRCReturn(rc, rc);
4188
4189	uint32_t u32Access = pSelReg->Attr.u;
4190	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4191	{
4192	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4193	u32Access = 0xf3;
4194	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4195	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4196	}
4197	else
4198	{
4199	/*
4200	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4201	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4202	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4203	* loaded in protected-mode have their attribute as 0.
4204	*/
4205	if (!u32Access)
4206	u32Access = X86DESCATTR_UNUSABLE;
4207	}
4208
4209	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4210	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4211	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4212
4213	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4214	AssertRCReturn(rc, rc);
4215	return rc;
4216	}
4217
4218
4219	/**
4220	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4221	* into the guest-state area in the VMCS.
4222	*
4223	* @returns VBox status code.
4224	* @param pVM Pointer to the VM.
4225	* @param pVCPU Pointer to the VMCPU.
4226	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4227	* out-of-sync. Make sure to update the required fields
4228	* before using them.
4229	*
4230	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4231	* @remarks No-long-jump zone!!!
4232	*/
4233	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4234	{
4235	int rc = VERR_INTERNAL_ERROR_5;
4236	PVM pVM = pVCpu->CTX_SUFF(pVM);
4237
4238	/*
4239	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4240	*/
4241	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4242	{
4243	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4244	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4245	{
4246	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4247	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4248	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4249	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4250	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4251	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4252	}
4253
4254	#ifdef VBOX_WITH_REM
4255	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4256	{
4257	Assert(pVM->hm.s.vmx.pRealModeTSS);
4258	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4259	if ( pVCpu->hm.s.vmx.fWasInRealMode
4260	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4261	{
4262	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4263	in real-mode (e.g. OpenBSD 4.0) */
4264	REMFlushTBs(pVM);
4265	Log4(("Load: Switch to protected mode detected!\n"));
4266	pVCpu->hm.s.vmx.fWasInRealMode = false;
4267	}
4268	}
4269	#endif
4270	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4271	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4272	AssertRCReturn(rc, rc);
4273	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4274	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4275	AssertRCReturn(rc, rc);
4276	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4277	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4278	AssertRCReturn(rc, rc);
4279	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4280	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4281	AssertRCReturn(rc, rc);
4282	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4283	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4284	AssertRCReturn(rc, rc);
4285	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4286	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4287	AssertRCReturn(rc, rc);
4288
4289	#ifdef VBOX_STRICT
4290	/* Validate. */
4291	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4292	#endif
4293
4294	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4295	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4296	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4297	}
4298
4299	/*
4300	* Guest TR.
4301	*/
4302	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4303	{
4304	/*
4305	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4306	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4307	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4308	*/
4309	uint16_t u16Sel = 0;
4310	uint32_t u32Limit = 0;
4311	uint64_t u64Base = 0;
4312	uint32_t u32AccessRights = 0;
4313
4314	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4315	{
4316	u16Sel = pMixedCtx->tr.Sel;
4317	u32Limit = pMixedCtx->tr.u32Limit;
4318	u64Base = pMixedCtx->tr.u64Base;
4319	u32AccessRights = pMixedCtx->tr.Attr.u;
4320	}
4321	else
4322	{
4323	Assert(pVM->hm.s.vmx.pRealModeTSS);
4324	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4325
4326	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4327	RTGCPHYS GCPhys;
4328	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4329	AssertRCReturn(rc, rc);
4330
4331	X86DESCATTR DescAttr;
4332	DescAttr.u = 0;
4333	DescAttr.n.u1Present = 1;
4334	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4335
4336	u16Sel = 0;
4337	u32Limit = HM_VTX_TSS_SIZE;
4338	u64Base = GCPhys; /* in real-mode phys = virt. */
4339	u32AccessRights = DescAttr.u;
4340	}
4341
4342	/* Validate. */
4343	Assert(!(u16Sel & RT_BIT(2)));
4344	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4345	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4346	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4347	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4348	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4349	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4350	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4351	Assert( (u32Limit & 0xfff) == 0xfff
4352	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4353	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4354	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4355
4356	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4357	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4358	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4359	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4360
4361	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4362	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4363	}
4364
4365	/*
4366	* Guest GDTR.
4367	*/
4368	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4369	{
4370	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4371	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4372
4373	/* Validate. */
4374	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4375
4376	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4377	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4378	}
4379
4380	/*
4381	* Guest LDTR.
4382	*/
4383	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4384	{
4385	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4386	uint32_t u32Access = 0;
4387	if (!pMixedCtx->ldtr.Attr.u)
4388	u32Access = X86DESCATTR_UNUSABLE;
4389	else
4390	u32Access = pMixedCtx->ldtr.Attr.u;
4391
4392	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4393	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4394	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4395	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4396
4397	/* Validate. */
4398	if (!(u32Access & X86DESCATTR_UNUSABLE))
4399	{
4400	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4401	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4402	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4403	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4404	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4405	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4406	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4407	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4408	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4409	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4410	}
4411
4412	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4413	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4414	}
4415
4416	/*
4417	* Guest IDTR.
4418	*/
4419	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4420	{
4421	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4422	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4423
4424	/* Validate. */
4425	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4426
4427	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4428	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4429	}
4430
4431	return VINF_SUCCESS;
4432	}
4433
4434
4435	/**
4436	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4437	* areas. These MSRs will automatically be loaded to the host CPU on every
4438	* successful VM entry and stored from the host CPU on every successful VM exit.
4439	*
4440	* This also creates/updates MSR slots for the host MSRs. The actual host
4441	* MSR values are -not- updated here for performance reasons. See
4442	* hmR0VmxSaveHostMsrs().
4443	*
4444	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4445	*
4446	* @returns VBox status code.
4447	* @param pVCpu Pointer to the VMCPU.
4448	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4449	* out-of-sync. Make sure to update the required fields
4450	* before using them.
4451	*
4452	* @remarks No-long-jump zone!!!
4453	*/
4454	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4455	{
4456	AssertPtr(pVCpu);
4457	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4458
4459	/*
4460	* MSRs that we use the auto-load/store MSR area in the VMCS.
4461	*/
4462	PVM pVM = pVCpu->CTX_SUFF(pVM);
4463	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4464	{
4465	if (pVM->hm.s.fAllow64BitGuests)
4466	{
4467	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4468	if (!HMVMX_IS_64BIT_HOST_MODE())
4469	{
4470	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4471	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4472	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4473	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4474	}
4475	# ifdef DEBUG
4476	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4477	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4478	Log4(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4479	# endif
4480	#endif
4481	}
4482	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4483	}
4484
4485	/*
4486	* Guest Sysenter MSRs.
4487	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4488	* VM-exits on WRMSRs for these MSRs.
4489	*/
4490	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4491	{
4492	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4493	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4494	}
4495
4496	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4497	{
4498	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4499	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4500	}
4501
4502	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4503	{
4504	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4505	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4506	}
4507
4508	return VINF_SUCCESS;
4509	}
4510
4511
4512	/**
4513	* Loads the guest activity state into the guest-state area in the VMCS.
4514	*
4515	* @returns VBox status code.
4516	* @param pVCpu Pointer to the VMCPU.
4517	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4518	* out-of-sync. Make sure to update the required fields
4519	* before using them.
4520	*
4521	* @remarks No-long-jump zone!!!
4522	*/
4523	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4524	{
4525	NOREF(pCtx);
4526	/** @todo See if we can make use of other states, e.g.
4527	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4528	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4529	{
4530	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4531	AssertRCReturn(rc, rc);
4532
4533	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4534	}
4535	return VINF_SUCCESS;
4536	}
4537
4538
4539	/**
4540	* Sets up the appropriate function to run guest code.
4541	*
4542	* @returns VBox status code.
4543	* @param pVCpu Pointer to the VMCPU.
4544	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4545	* out-of-sync. Make sure to update the required fields
4546	* before using them.
4547	*
4548	* @remarks No-long-jump zone!!!
4549	*/
4550	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4551	{
4552	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4553	{
4554	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4555	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4556	#endif
4557	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4558	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4559	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4560	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4561	{
4562	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4563	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4564	}
4565	#else
4566	/* 64-bit host or hybrid host. */
4567	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4568	#endif
4569	}
4570	else
4571	{
4572	/* Guest is not in long mode, use the 32-bit handler. */
4573	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4574	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4575	{
4576	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4577	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4578	}
4579	#else
4580	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4581	#endif
4582	}
4583	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4584	return VINF_SUCCESS;
4585	}
4586
4587
4588	/**
4589	* Wrapper for running the guest code in VT-x.
4590	*
4591	* @returns VBox strict status code.
4592	* @param pVM Pointer to the VM.
4593	* @param pVCpu Pointer to the VMCPU.
4594	* @param pCtx Pointer to the guest-CPU context.
4595	*
4596	* @remarks No-long-jump zone!!!
4597	*/
4598	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4599	{
4600	/*
4601	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4602	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4603	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4604	*/
4605	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4606	/** @todo Add stats for resume vs launch. */
4607	#ifdef VBOX_WITH_KERNEL_USING_XMM
4608	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4609	#else
4610	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4611	#endif
4612	}
4613
4614
4615	/**
4616	* Reports world-switch error and dumps some useful debug info.
4617	*
4618	* @param pVM Pointer to the VM.
4619	* @param pVCpu Pointer to the VMCPU.
4620	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4621	* @param pCtx Pointer to the guest-CPU context.
4622	* @param pVmxTransient Pointer to the VMX transient structure (only
4623	* exitReason updated).
4624	*/
4625	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4626	{
4627	Assert(pVM);
4628	Assert(pVCpu);
4629	Assert(pCtx);
4630	Assert(pVmxTransient);
4631	HMVMX_ASSERT_PREEMPT_SAFE();
4632
4633	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4634	switch (rcVMRun)
4635	{
4636	case VERR_VMX_INVALID_VMXON_PTR:
4637	AssertFailed();
4638	break;
4639	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4640	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4641	{
4642	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4643	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4644	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4645	AssertRC(rc);
4646
4647	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4648	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4649	Cannot do it here as we may have been long preempted. */
4650
4651	#ifdef VBOX_STRICT
4652	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4653	pVmxTransient->uExitReason));
4654	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4655	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4656	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4657	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4658	else
4659	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4660	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4661	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4662
4663	/* VMX control bits. */
4664	uint32_t u32Val;
4665	uint64_t u64Val;
4666	HMVMXHCUINTREG uHCReg;
4667	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4668	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4669	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4670	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4671	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4672	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4673	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4674	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4675	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4676	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4677	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4678	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4679	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4680	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4681	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4682	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4683	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4684	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4685	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4686	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4687	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4688	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4689	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4690	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4691	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4692	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4693	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4694	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4695	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4696	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4697	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4698	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4699	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4700	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4701	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4702	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4703	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4704	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4705	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4706	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4707	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4708	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4709
4710	/* Guest bits. */
4711	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4712	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4713	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4714	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4715	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4716	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4717	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4718	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4719
4720	/* Host bits. */
4721	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4722	Log4(("Host CR0 %#RHr\n", uHCReg));
4723	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4724	Log4(("Host CR3 %#RHr\n", uHCReg));
4725	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4726	Log4(("Host CR4 %#RHr\n", uHCReg));
4727
4728	RTGDTR HostGdtr;
4729	PCX86DESCHC pDesc;
4730	ASMGetGDTR(&HostGdtr);
4731	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4732	Log4(("Host CS %#08x\n", u32Val));
4733	if (u32Val < HostGdtr.cbGdt)
4734	{
4735	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4736	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4737	}
4738
4739	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4740	Log4(("Host DS %#08x\n", u32Val));
4741	if (u32Val < HostGdtr.cbGdt)
4742	{
4743	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4744	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4745	}
4746
4747	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4748	Log4(("Host ES %#08x\n", u32Val));
4749	if (u32Val < HostGdtr.cbGdt)
4750	{
4751	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4752	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4753	}
4754
4755	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4756	Log4(("Host FS %#08x\n", u32Val));
4757	if (u32Val < HostGdtr.cbGdt)
4758	{
4759	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4760	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4761	}
4762
4763	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4764	Log4(("Host GS %#08x\n", u32Val));
4765	if (u32Val < HostGdtr.cbGdt)
4766	{
4767	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4768	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4769	}
4770
4771	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4772	Log4(("Host SS %#08x\n", u32Val));
4773	if (u32Val < HostGdtr.cbGdt)
4774	{
4775	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4776	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4777	}
4778
4779	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4780	Log4(("Host TR %#08x\n", u32Val));
4781	if (u32Val < HostGdtr.cbGdt)
4782	{
4783	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4784	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4785	}
4786
4787	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4788	Log4(("Host TR Base %#RHv\n", uHCReg));
4789	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4790	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4791	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4792	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4793	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4794	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4795	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4796	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4797	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4798	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4799	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4800	Log4(("Host RSP %#RHv\n", uHCReg));
4801	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4802	Log4(("Host RIP %#RHv\n", uHCReg));
4803	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4804	if (HMVMX_IS_64BIT_HOST_MODE())
4805	{
4806	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4807	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4808	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4809	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4810	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4811	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4812	}
4813	# endif
4814	#endif /* VBOX_STRICT */
4815	break;
4816	}
4817
4818	default:
4819	/* Impossible */
4820	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4821	break;
4822	}
4823	NOREF(pVM); NOREF(pCtx);
4824	}
4825
4826
4827	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4828	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4829	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4830	#endif
4831	#ifdef VBOX_STRICT
4832	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4833	{
4834	switch (idxField)
4835	{
4836	case VMX_VMCS_GUEST_RIP:
4837	case VMX_VMCS_GUEST_RSP:
4838	case VMX_VMCS_GUEST_SYSENTER_EIP:
4839	case VMX_VMCS_GUEST_SYSENTER_ESP:
4840	case VMX_VMCS_GUEST_GDTR_BASE:
4841	case VMX_VMCS_GUEST_IDTR_BASE:
4842	case VMX_VMCS_GUEST_CS_BASE:
4843	case VMX_VMCS_GUEST_DS_BASE:
4844	case VMX_VMCS_GUEST_ES_BASE:
4845	case VMX_VMCS_GUEST_FS_BASE:
4846	case VMX_VMCS_GUEST_GS_BASE:
4847	case VMX_VMCS_GUEST_SS_BASE:
4848	case VMX_VMCS_GUEST_LDTR_BASE:
4849	case VMX_VMCS_GUEST_TR_BASE:
4850	case VMX_VMCS_GUEST_CR3:
4851	return true;
4852	}
4853	return false;
4854	}
4855
4856	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4857	{
4858	switch (idxField)
4859	{
4860	/* Read-only fields. */
4861	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4862	return true;
4863	}
4864	/* Remaining readable fields should also be writable. */
4865	return hmR0VmxIsValidWriteField(idxField);
4866	}
4867	#endif /* VBOX_STRICT */
4868
4869
4870	/**
4871	* Executes the specified handler in 64-bit mode.
4872	*
4873	* @returns VBox status code.
4874	* @param pVM Pointer to the VM.
4875	* @param pVCpu Pointer to the VMCPU.
4876	* @param pCtx Pointer to the guest CPU context.
4877	* @param enmOp The operation to perform.
4878	* @param cbParam Number of parameters.
4879	* @param paParam Array of 32-bit parameters.
4880	*/
4881	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4882	uint32_t *paParam)
4883	{
4884	int rc, rc2;
4885	PHMGLOBALCPUINFO pCpu;
4886	RTHCPHYS HCPhysCpuPage;
4887	RTCCUINTREG uOldEflags;
4888
4889	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4890	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4891	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4892	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4893
4894	#ifdef VBOX_STRICT
4895	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4896	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4897
4898	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4899	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4900	#endif
4901
4902	/* Disable interrupts. */
4903	uOldEflags = ASMIntDisableFlags();
4904
4905	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4906	RTCPUID idHostCpu = RTMpCpuId();
4907	CPUMR0SetLApic(pVCpu, idHostCpu);
4908	#endif
4909
4910	pCpu = HMR0GetCurrentCpu();
4911	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4912
4913	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4914	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4915
4916	/* Leave VMX Root Mode. */
4917	VMXDisable();
4918
4919	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4920
4921	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4922	CPUMSetHyperEIP(pVCpu, enmOp);
4923	for (int i = (int)cbParam - 1; i >= 0; i--)
4924	CPUMPushHyper(pVCpu, paParam[i]);
4925
4926	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4927
4928	/* Call the switcher. */
4929	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4930	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4931
4932	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4933	/* Make sure the VMX instructions don't cause #UD faults. */
4934	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4935
4936	/* Re-enter VMX Root Mode */
4937	rc2 = VMXEnable(HCPhysCpuPage);
4938	if (RT_FAILURE(rc2))
4939	{
4940	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4941	ASMSetFlags(uOldEflags);
4942	return rc2;
4943	}
4944
4945	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4946	AssertRC(rc2);
4947	Assert(!(ASMGetFlags() & X86_EFL_IF));
4948	ASMSetFlags(uOldEflags);
4949	return rc;
4950	}
4951
4952
4953	/**
4954	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4955	* supporting 64-bit guests.
4956	*
4957	* @returns VBox status code.
4958	* @param fResume Whether to VMLAUNCH or VMRESUME.
4959	* @param pCtx Pointer to the guest-CPU context.
4960	* @param pCache Pointer to the VMCS cache.
4961	* @param pVM Pointer to the VM.
4962	* @param pVCpu Pointer to the VMCPU.
4963	*/
4964	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4965	{
4966	uint32_t aParam[6];
4967	PHMGLOBALCPUINFO pCpu = NULL;
4968	RTHCPHYS HCPhysCpuPage = 0;
4969	int rc = VERR_INTERNAL_ERROR_5;
4970
4971	pCpu = HMR0GetCurrentCpu();
4972	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4973
4974	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4975	pCache->uPos = 1;
4976	pCache->interPD = PGMGetInterPaeCR3(pVM);
4977	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4978	#endif
4979
4980	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4981	pCache->TestIn.HCPhysCpuPage = 0;
4982	pCache->TestIn.HCPhysVmcs = 0;
4983	pCache->TestIn.pCache = 0;
4984	pCache->TestOut.HCPhysVmcs = 0;
4985	pCache->TestOut.pCache = 0;
4986	pCache->TestOut.pCtx = 0;
4987	pCache->TestOut.eflags = 0;
4988	#endif
4989
4990	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4991	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4992	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4993	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4994	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4995	aParam[5] = 0;
4996
4997	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4998	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4999	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5000	#endif
5001	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5002
5003	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5004	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5005	Assert(pCtx->dr[4] == 10);
5006	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5007	#endif
5008
5009	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5010	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5011	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5012	pVCpu->hm.s.vmx.HCPhysVmcs));
5013	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5014	pCache->TestOut.HCPhysVmcs));
5015	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5016	pCache->TestOut.pCache));
5017	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5018	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5019	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5020	pCache->TestOut.pCtx));
5021	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5022	#endif
5023	return rc;
5024	}
5025
5026
5027	/**
5028	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5029	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5030	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5031	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5032	*
5033	* @returns VBox status code.
5034	* @param pVM Pointer to the VM.
5035	* @param pVCpu Pointer to the VMCPU.
5036	*/
5037	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5038	{
5039	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5040	{ \
5041	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5042	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5043	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5044	++cReadFields; \
5045	}
5046
5047	AssertPtr(pVM);
5048	AssertPtr(pVCpu);
5049	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5050	uint32_t cReadFields = 0;
5051
5052	/*
5053	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5054	* and serve to indicate exceptions to the rules.
5055	*/
5056
5057	/* Guest-natural selector base fields. */
5058	#if 0
5059	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5062	#endif
5063	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5064	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5066	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5067	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5069	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5070	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5071	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5072	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5075	#if 0
5076	/* Unused natural width guest-state fields. */
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5079	#endif
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5081	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5082
5083	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5084	#if 0
5085	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5086	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5087	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5088	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5089	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5090	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5091	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5092	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5093	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5094	#endif
5095
5096	/* Natural width guest-state fields. */
5097	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5098	#if 0
5099	/* Currently unused field. */
5100	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5101	#endif
5102
5103	if (pVM->hm.s.fNestedPaging)
5104	{
5105	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5106	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5107	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5108	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5109	}
5110	else
5111	{
5112	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5113	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5114	}
5115
5116	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5117	return VINF_SUCCESS;
5118	}
5119
5120
5121	/**
5122	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5123	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5124	* darwin, running 64-bit guests).
5125	*
5126	* @returns VBox status code.
5127	* @param pVCpu Pointer to the VMCPU.
5128	* @param idxField The VMCS field encoding.
5129	* @param u64Val 16, 32 or 64-bit value.
5130	*/
5131	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5132	{
5133	int rc;
5134	switch (idxField)
5135	{
5136	/*
5137	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5138	*/
5139	/* 64-bit Control fields. */
5140	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5141	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5142	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5143	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5144	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5145	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5146	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5147	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5148	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5149	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5150	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5151	case VMX_VMCS64_CTRL_EPTP_FULL:
5152	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5153	/* 64-bit Guest-state fields. */
5154	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5155	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5156	case VMX_VMCS64_GUEST_PAT_FULL:
5157	case VMX_VMCS64_GUEST_EFER_FULL:
5158	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5159	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5160	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5161	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5162	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5163	/* 64-bit Host-state fields. */
5164	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5165	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5166	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5167	{
5168	rc = VMXWriteVmcs32(idxField, u64Val);
5169	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5170	break;
5171	}
5172
5173	/*
5174	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5175	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5176	*/
5177	/* Natural-width Guest-state fields. */
5178	case VMX_VMCS_GUEST_CR3:
5179	case VMX_VMCS_GUEST_ES_BASE:
5180	case VMX_VMCS_GUEST_CS_BASE:
5181	case VMX_VMCS_GUEST_SS_BASE:
5182	case VMX_VMCS_GUEST_DS_BASE:
5183	case VMX_VMCS_GUEST_FS_BASE:
5184	case VMX_VMCS_GUEST_GS_BASE:
5185	case VMX_VMCS_GUEST_LDTR_BASE:
5186	case VMX_VMCS_GUEST_TR_BASE:
5187	case VMX_VMCS_GUEST_GDTR_BASE:
5188	case VMX_VMCS_GUEST_IDTR_BASE:
5189	case VMX_VMCS_GUEST_RSP:
5190	case VMX_VMCS_GUEST_RIP:
5191	case VMX_VMCS_GUEST_SYSENTER_ESP:
5192	case VMX_VMCS_GUEST_SYSENTER_EIP:
5193	{
5194	if (!(u64Val >> 32))
5195	{
5196	/* If this field is 64-bit, VT-x will zero out the top bits. */
5197	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5198	}
5199	else
5200	{
5201	/* Assert that only the 32->64 switcher case should ever come here. */
5202	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5203	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5204	}
5205	break;
5206	}
5207
5208	default:
5209	{
5210	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5211	rc = VERR_INVALID_PARAMETER;
5212	break;
5213	}
5214	}
5215	AssertRCReturn(rc, rc);
5216	return rc;
5217	}
5218
5219
5220	/**
5221	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5222	* hosts (except darwin) for 64-bit guests.
5223	*
5224	* @param pVCpu Pointer to the VMCPU.
5225	* @param idxField The VMCS field encoding.
5226	* @param u64Val 16, 32 or 64-bit value.
5227	*/
5228	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5229	{
5230	AssertPtr(pVCpu);
5231	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5232
5233	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5234	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5235
5236	/* Make sure there are no duplicates. */
5237	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5238	{
5239	if (pCache->Write.aField[i] == idxField)
5240	{
5241	pCache->Write.aFieldVal[i] = u64Val;
5242	return VINF_SUCCESS;
5243	}
5244	}
5245
5246	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5247	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5248	pCache->Write.cValidEntries++;
5249	return VINF_SUCCESS;
5250	}
5251
5252	/* Enable later when the assembly code uses these as callbacks. */
5253	#if 0
5254	/*
5255	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5256	*
5257	* @param pVCpu Pointer to the VMCPU.
5258	* @param pCache Pointer to the VMCS cache.
5259	*
5260	* @remarks No-long-jump zone!!!
5261	*/
5262	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5263	{
5264	AssertPtr(pCache);
5265	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5266	{
5267	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5268	AssertRC(rc);
5269	}
5270	pCache->Write.cValidEntries = 0;
5271	}
5272
5273
5274	/**
5275	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5276	*
5277	* @param pVCpu Pointer to the VMCPU.
5278	* @param pCache Pointer to the VMCS cache.
5279	*
5280	* @remarks No-long-jump zone!!!
5281	*/
5282	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5283	{
5284	AssertPtr(pCache);
5285	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5286	{
5287	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5288	AssertRC(rc);
5289	}
5290	}
5291	#endif
5292	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5293
5294
5295	/**
5296	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5297	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5298	* timer.
5299	*
5300	* @returns VBox status code.
5301	* @param pVCpu Pointer to the VMCPU.
5302	*
5303	* @remarks No-long-jump zone!!!
5304	*/
5305	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5306	{
5307	int rc = VERR_INTERNAL_ERROR_5;
5308	bool fOffsettedTsc = false;
5309	PVM pVM = pVCpu->CTX_SUFF(pVM);
5310	if (pVM->hm.s.vmx.fUsePreemptTimer)
5311	{
5312	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5313
5314	/* Make sure the returned values have sane upper and lower boundaries. */
5315	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5316	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5317	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5318	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5319
5320	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5321	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5322	}
5323	else
5324	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5325
5326	if (fOffsettedTsc)
5327	{
5328	uint64_t u64CurTSC = ASMReadTSC();
5329	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5330	{
5331	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5332	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5333
5334	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5335	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5336	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5337	}
5338	else
5339	{
5340	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5341	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5342	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5343	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5344	}
5345	}
5346	else
5347	{
5348	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5349	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5350	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5351	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5352	}
5353	}
5354
5355
5356	/**
5357	* Determines if an exception is a contributory exception. Contributory
5358	* exceptions are ones which can cause double-faults. Page-fault is
5359	* intentionally not included here as it's a conditional contributory exception.
5360	*
5361	* @returns true if the exception is contributory, false otherwise.
5362	* @param uVector The exception vector.
5363	*/
5364	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5365	{
5366	switch (uVector)
5367	{
5368	case X86_XCPT_GP:
5369	case X86_XCPT_SS:
5370	case X86_XCPT_NP:
5371	case X86_XCPT_TS:
5372	case X86_XCPT_DE:
5373	return true;
5374	default:
5375	break;
5376	}
5377	return false;
5378	}
5379
5380
5381	/**
5382	* Sets an event as a pending event to be injected into the guest.
5383	*
5384	* @param pVCpu Pointer to the VMCPU.
5385	* @param u32IntInfo The VM-entry interruption-information field.
5386	* @param cbInstr The VM-entry instruction length in bytes (for software
5387	* interrupts, exceptions and privileged software
5388	* exceptions).
5389	* @param u32ErrCode The VM-entry exception error code.
5390	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5391	* page-fault.
5392	*
5393	* @remarks Statistics counter assumes this is a guest event being injected or
5394	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5395	* always incremented.
5396	*/
5397	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5398	RTGCUINTPTR GCPtrFaultAddress)
5399	{
5400	Assert(!pVCpu->hm.s.Event.fPending);
5401	pVCpu->hm.s.Event.fPending = true;
5402	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5403	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5404	pVCpu->hm.s.Event.cbInstr = cbInstr;
5405	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5406
5407	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5408	}
5409
5410
5411	/**
5412	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5413	*
5414	* @param pVCpu Pointer to the VMCPU.
5415	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5416	* out-of-sync. Make sure to update the required fields
5417	* before using them.
5418	*/
5419	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5420	{
5421	NOREF(pMixedCtx);
5422	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5423	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5424	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5425	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5426	}
5427
5428
5429	/**
5430	* Handle a condition that occurred while delivering an event through the guest
5431	* IDT.
5432	*
5433	* @returns VBox status code (informational error codes included).
5434	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5435	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5436	* continue execution of the guest which will delivery the #DF.
5437	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5438	*
5439	* @param pVCpu Pointer to the VMCPU.
5440	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5441	* out-of-sync. Make sure to update the required fields
5442	* before using them.
5443	* @param pVmxTransient Pointer to the VMX transient structure.
5444	*
5445	* @remarks No-long-jump zone!!!
5446	*/
5447	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5448	{
5449	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5450	AssertRCReturn(rc, rc);
5451	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5452	{
5453	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5454	AssertRCReturn(rc, rc);
5455
5456	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5457	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5458	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5459
5460	typedef enum
5461	{
5462	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5463	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5464	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5465	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5466	} VMXREFLECTXCPT;
5467
5468	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5469	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5470	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5471	{
5472	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5473	{
5474	enmReflect = VMXREFLECTXCPT_XCPT;
5475	#ifdef VBOX_STRICT
5476	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5477	&& uExitVector == X86_XCPT_PF)
5478	{
5479	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5480	}
5481	#endif
5482	if ( uExitVector == X86_XCPT_PF
5483	&& uIdtVector == X86_XCPT_PF)
5484	{
5485	pVmxTransient->fVectoringPF = true;
5486	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5487	}
5488	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5489	&& hmR0VmxIsContributoryXcpt(uExitVector)
5490	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5491	\|\| uIdtVector == X86_XCPT_PF))
5492	{
5493	enmReflect = VMXREFLECTXCPT_DF;
5494	}
5495	else if (uIdtVector == X86_XCPT_DF)
5496	enmReflect = VMXREFLECTXCPT_TF;
5497	}
5498	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5499	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5500	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5501	{
5502	/*
5503	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5504	* (whatever they are) as they reoccur when restarting the instruction.
5505	*/
5506	enmReflect = VMXREFLECTXCPT_XCPT;
5507	}
5508	}
5509	else
5510	{
5511	/*
5512	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5513	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5514	* original exception to the guest after handling the VM-exit.
5515	*/
5516	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5517	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5518	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5519	{
5520	enmReflect = VMXREFLECTXCPT_XCPT;
5521	}
5522	}
5523
5524	switch (enmReflect)
5525	{
5526	case VMXREFLECTXCPT_XCPT:
5527	{
5528	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5529	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5530	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5531
5532	uint32_t u32ErrCode = 0;
5533	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5534	{
5535	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5536	AssertRCReturn(rc, rc);
5537	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5538	}
5539
5540	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5541	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5542	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5543	rc = VINF_SUCCESS;
5544	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5545	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5546
5547	break;
5548	}
5549
5550	case VMXREFLECTXCPT_DF:
5551	{
5552	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5553	rc = VINF_HM_DOUBLE_FAULT;
5554	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5555	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5556
5557	break;
5558	}
5559
5560	case VMXREFLECTXCPT_TF:
5561	{
5562	rc = VINF_EM_RESET;
5563	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5564	uExitVector));
5565	break;
5566	}
5567
5568	default:
5569	Assert(rc == VINF_SUCCESS);
5570	break;
5571	}
5572	}
5573	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5574	return rc;
5575	}
5576
5577
5578	/**
5579	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5580	*
5581	* @returns VBox status code.
5582	* @param pVCpu Pointer to the VMCPU.
5583	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5584	* out-of-sync. Make sure to update the required fields
5585	* before using them.
5586	*
5587	* @remarks No-long-jump zone!!!
5588	*/
5589	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5590	{
5591	NOREF(pMixedCtx);
5592
5593	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5594	{
5595	uint32_t uVal = 0;
5596	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5597	AssertRCReturn(rc, rc);
5598
5599	uint32_t uShadow = 0;
5600	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5601	AssertRCReturn(rc, rc);
5602
5603	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5604	CPUMSetGuestCR0(pVCpu, uVal);
5605	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5606	}
5607	return VINF_SUCCESS;
5608	}
5609
5610
5611	/**
5612	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5613	*
5614	* @returns VBox status code.
5615	* @param pVCpu Pointer to the VMCPU.
5616	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5617	* out-of-sync. Make sure to update the required fields
5618	* before using them.
5619	*
5620	* @remarks No-long-jump zone!!!
5621	*/
5622	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5623	{
5624	NOREF(pMixedCtx);
5625
5626	int rc = VINF_SUCCESS;
5627	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5628	{
5629	uint32_t uVal = 0;
5630	uint32_t uShadow = 0;
5631	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5632	AssertRCReturn(rc, rc);
5633	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5634	AssertRCReturn(rc, rc);
5635
5636	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5637	CPUMSetGuestCR4(pVCpu, uVal);
5638	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5639	}
5640	return rc;
5641	}
5642
5643
5644	/**
5645	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5646	*
5647	* @returns VBox status code.
5648	* @param pVCpu Pointer to the VMCPU.
5649	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5650	* out-of-sync. Make sure to update the required fields
5651	* before using them.
5652	*
5653	* @remarks No-long-jump zone!!!
5654	*/
5655	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5656	{
5657	int rc = VINF_SUCCESS;
5658	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5659	{
5660	uint64_t u64Val = 0;
5661	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5662	AssertRCReturn(rc, rc);
5663
5664	pMixedCtx->rip = u64Val;
5665	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5666	}
5667	return rc;
5668	}
5669
5670
5671	/**
5672	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5673	*
5674	* @returns VBox status code.
5675	* @param pVCpu Pointer to the VMCPU.
5676	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5677	* out-of-sync. Make sure to update the required fields
5678	* before using them.
5679	*
5680	* @remarks No-long-jump zone!!!
5681	*/
5682	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5683	{
5684	int rc = VINF_SUCCESS;
5685	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5686	{
5687	uint64_t u64Val = 0;
5688	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5689	AssertRCReturn(rc, rc);
5690
5691	pMixedCtx->rsp = u64Val;
5692	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5693	}
5694	return rc;
5695	}
5696
5697
5698	/**
5699	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5700	*
5701	* @returns VBox status code.
5702	* @param pVCpu Pointer to the VMCPU.
5703	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5704	* out-of-sync. Make sure to update the required fields
5705	* before using them.
5706	*
5707	* @remarks No-long-jump zone!!!
5708	*/
5709	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5710	{
5711	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5712	{
5713	uint32_t uVal = 0;
5714	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5715	AssertRCReturn(rc, rc);
5716
5717	pMixedCtx->eflags.u32 = uVal;
5718	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5719	{
5720	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5721	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5722
5723	pMixedCtx->eflags.Bits.u1VM = 0;
5724	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5725	}
5726
5727	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5728	}
5729	return VINF_SUCCESS;
5730	}
5731
5732
5733	/**
5734	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5735	* guest-CPU context.
5736	*/
5737	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5738	{
5739	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5740	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5741	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5742	return rc;
5743	}
5744
5745
5746	/**
5747	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5748	* from the guest-state area in the VMCS.
5749	*
5750	* @param pVCpu Pointer to the VMCPU.
5751	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5752	* out-of-sync. Make sure to update the required fields
5753	* before using them.
5754	*
5755	* @remarks No-long-jump zone!!!
5756	*/
5757	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5758	{
5759	uint32_t uIntrState = 0;
5760	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5761	AssertRC(rc);
5762
5763	if (!uIntrState)
5764	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5765	else
5766	{
5767	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5768	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5769	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5770	AssertRC(rc);
5771	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5772	AssertRC(rc);
5773
5774	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5775	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5776	}
5777	}
5778
5779
5780	/**
5781	* Saves the guest's activity state.
5782	*
5783	* @returns VBox status code.
5784	* @param pVCpu Pointer to the VMCPU.
5785	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5786	* out-of-sync. Make sure to update the required fields
5787	* before using them.
5788	*
5789	* @remarks No-long-jump zone!!!
5790	*/
5791	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5792	{
5793	NOREF(pMixedCtx);
5794	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5795	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5796	return VINF_SUCCESS;
5797	}
5798
5799
5800	/**
5801	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5802	* the current VMCS into the guest-CPU context.
5803	*
5804	* @returns VBox status code.
5805	* @param pVCpu Pointer to the VMCPU.
5806	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5807	* out-of-sync. Make sure to update the required fields
5808	* before using them.
5809	*
5810	* @remarks No-long-jump zone!!!
5811	*/
5812	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5813	{
5814	int rc = VINF_SUCCESS;
5815	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5816	{
5817	uint32_t u32Val = 0;
5818	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5819	pMixedCtx->SysEnter.cs = u32Val;
5820	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5821	}
5822
5823	uint64_t u64Val = 0;
5824	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5825	{
5826	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5827	pMixedCtx->SysEnter.eip = u64Val;
5828	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5829	}
5830	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5831	{
5832	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5833	pMixedCtx->SysEnter.esp = u64Val;
5834	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5835	}
5836	return rc;
5837	}
5838
5839
5840	/**
5841	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5842	* the CPU back into the guest-CPU context.
5843	*
5844	* @returns VBox status code.
5845	* @param pVCpu Pointer to the VMCPU.
5846	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5847	* out-of-sync. Make sure to update the required fields
5848	* before using them.
5849	*
5850	* @remarks No-long-jump zone!!!
5851	*/
5852	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5853	{
5854	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5855	if ( HMVMX_IS_64BIT_HOST_MODE()
5856	&& pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5857	{
5858	/* We should not get preempted to a different CPU at this point while reading the MSRs. */
5859	VMMRZCallRing3Disable(pVCpu);
5860	HM_DISABLE_PREEMPT_IF_NEEDED();
5861
5862	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5863	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS))
5864	{
5865	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5866	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5867	}
5868
5869	HM_RESTORE_PREEMPT_IF_NEEDED();
5870	VMMRZCallRing3Enable(pVCpu);
5871	}
5872	else
5873	{
5874	/* Darwin 32-bit/PAE kernel or 64-bit host running 32-bit guest. */
5875	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5876	}
5877	#else /* HC_ARCH_BITS == 32 */
5878	NOREF(pMixedCtx);
5879	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5880	#endif
5881
5882	return VINF_SUCCESS;
5883	}
5884
5885
5886	/**
5887	* Saves the auto load/store'd guest MSRs from the current VMCS into
5888	* the guest-CPU context.
5889	*
5890	* @returns VBox status code.
5891	* @param pVCpu Pointer to the VMCPU.
5892	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5893	* out-of-sync. Make sure to update the required fields
5894	* before using them.
5895	*
5896	* @remarks No-long-jump zone!!!
5897	*/
5898	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5899	{
5900	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5901	return VINF_SUCCESS;
5902
5903	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5904	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5905	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5906	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5907	{
5908	switch (pMsr->u32Msr)
5909	{
5910	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5911	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5912	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5913	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5914	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5915	default:
5916	{
5917	AssertFailed();
5918	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5919	}
5920	}
5921	}
5922
5923	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5924	return VINF_SUCCESS;
5925	}
5926
5927
5928	/**
5929	* Saves the guest control registers from the current VMCS into the guest-CPU
5930	* context.
5931	*
5932	* @returns VBox status code.
5933	* @param pVCpu Pointer to the VMCPU.
5934	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5935	* out-of-sync. Make sure to update the required fields
5936	* before using them.
5937	*
5938	* @remarks No-long-jump zone!!!
5939	*/
5940	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5941	{
5942	/* Guest CR0. Guest FPU. */
5943	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5944	AssertRCReturn(rc, rc);
5945
5946	/* Guest CR4. */
5947	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5948	AssertRCReturn(rc, rc);
5949
5950	/* Guest CR2 - updated always during the world-switch or in #PF. */
5951	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5952	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5953	{
5954	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5955	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5956
5957	PVM pVM = pVCpu->CTX_SUFF(pVM);
5958	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5959	\|\| ( pVM->hm.s.fNestedPaging
5960	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5961	{
5962	uint64_t u64Val = 0;
5963	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5964	if (pMixedCtx->cr3 != u64Val)
5965	{
5966	CPUMSetGuestCR3(pVCpu, u64Val);
5967	if (VMMRZCallRing3IsEnabled(pVCpu))
5968	{
5969	PGMUpdateCR3(pVCpu, u64Val);
5970	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5971	}
5972	else
5973	{
5974	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5975	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5976	}
5977	}
5978
5979	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5980	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5981	{
5982	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5983	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5984	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5985	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5986
5987	if (VMMRZCallRing3IsEnabled(pVCpu))
5988	{
5989	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5990	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5991	}
5992	else
5993	{
5994	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5995	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5996	}
5997	}
5998	}
5999
6000	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
6001	}
6002
6003	/*
6004	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6005	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6006	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6007	*
6008	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6009	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6010	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6011	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6012	*
6013	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6014	*/
6015	if (VMMRZCallRing3IsEnabled(pVCpu))
6016	{
6017	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6018	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6019
6020	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6021	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6022
6023	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6024	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6025	}
6026
6027	return rc;
6028	}
6029
6030
6031	/**
6032	* Reads a guest segment register from the current VMCS into the guest-CPU
6033	* context.
6034	*
6035	* @returns VBox status code.
6036	* @param pVCpu Pointer to the VMCPU.
6037	* @param idxSel Index of the selector in the VMCS.
6038	* @param idxLimit Index of the segment limit in the VMCS.
6039	* @param idxBase Index of the segment base in the VMCS.
6040	* @param idxAccess Index of the access rights of the segment in the VMCS.
6041	* @param pSelReg Pointer to the segment selector.
6042	*
6043	* @remarks No-long-jump zone!!!
6044	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6045	* macro as that takes care of whether to read from the VMCS cache or
6046	* not.
6047	*/
6048	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6049	PCPUMSELREG pSelReg)
6050	{
6051	NOREF(pVCpu);
6052
6053	uint32_t u32Val = 0;
6054	int rc = VMXReadVmcs32(idxSel, &u32Val);
6055	AssertRCReturn(rc, rc);
6056	pSelReg->Sel = (uint16_t)u32Val;
6057	pSelReg->ValidSel = (uint16_t)u32Val;
6058	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6059
6060	rc = VMXReadVmcs32(idxLimit, &u32Val);
6061	AssertRCReturn(rc, rc);
6062	pSelReg->u32Limit = u32Val;
6063
6064	uint64_t u64Val = 0;
6065	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6066	AssertRCReturn(rc, rc);
6067	pSelReg->u64Base = u64Val;
6068
6069	rc = VMXReadVmcs32(idxAccess, &u32Val);
6070	AssertRCReturn(rc, rc);
6071	pSelReg->Attr.u = u32Val;
6072
6073	/*
6074	* If VT-x marks the segment as unusable, most other bits remain undefined:
6075	* - For CS the L, D and G bits have meaning.
6076	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6077	* - For the remaining data segments no bits are defined.
6078	*
6079	* The present bit and the unusable bit has been observed to be set at the
6080	* same time (the selector was supposed to invalid as we started executing
6081	* a V8086 interrupt in ring-0).
6082	*
6083	* What should be important for the rest of the VBox code that the P bit is
6084	* cleared. Some of the other VBox code recognizes the unusable bit, but
6085	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6086	* safe side here, we'll strip off P and other bits we don't care about. If
6087	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6088	*
6089	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6090	*/
6091	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6092	{
6093	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6094
6095	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6096	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6097	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6098
6099	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6100	#ifdef DEBUG_bird
6101	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6102	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6103	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6104	#endif
6105	}
6106	return VINF_SUCCESS;
6107	}
6108
6109
6110	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6111	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6112	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6113	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6114	#else
6115	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6116	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6117	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6118	#endif
6119
6120
6121	/**
6122	* Saves the guest segment registers from the current VMCS into the guest-CPU
6123	* context.
6124	*
6125	* @returns VBox status code.
6126	* @param pVCpu Pointer to the VMCPU.
6127	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6128	* out-of-sync. Make sure to update the required fields
6129	* before using them.
6130	*
6131	* @remarks No-long-jump zone!!!
6132	*/
6133	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6134	{
6135	/* Guest segment registers. */
6136	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6137	{
6138	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6139	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6140	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6141	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6142	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6143	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6144	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6145
6146	/* Restore segment attributes for real-on-v86 mode hack. */
6147	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6148	{
6149	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6150	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6151	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6152	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6153	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6154	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6155	}
6156	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
6157	}
6158
6159	return VINF_SUCCESS;
6160	}
6161
6162
6163	/**
6164	* Saves the guest descriptor table registers and task register from the current
6165	* VMCS into the guest-CPU context.
6166	*
6167	* @returns VBox status code.
6168	* @param pVCpu Pointer to the VMCPU.
6169	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6170	* out-of-sync. Make sure to update the required fields
6171	* before using them.
6172	*
6173	* @remarks No-long-jump zone!!!
6174	*/
6175	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6176	{
6177	int rc = VINF_SUCCESS;
6178
6179	/* Guest LDTR. */
6180	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
6181	{
6182	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6183	AssertRCReturn(rc, rc);
6184	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
6185	}
6186
6187	/* Guest GDTR. */
6188	uint64_t u64Val = 0;
6189	uint32_t u32Val = 0;
6190	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6191	{
6192	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6193	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6194	pMixedCtx->gdtr.pGdt = u64Val;
6195	pMixedCtx->gdtr.cbGdt = u32Val;
6196	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
6197	}
6198
6199	/* Guest IDTR. */
6200	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6201	{
6202	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6203	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6204	pMixedCtx->idtr.pIdt = u64Val;
6205	pMixedCtx->idtr.cbIdt = u32Val;
6206	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
6207	}
6208
6209	/* Guest TR. */
6210	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6211	{
6212	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6213	AssertRCReturn(rc, rc);
6214
6215	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6216	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6217	{
6218	rc = VMXLOCAL_READ_SEG(TR, tr);
6219	AssertRCReturn(rc, rc);
6220	}
6221	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
6222	}
6223	return rc;
6224	}
6225
6226	#undef VMXLOCAL_READ_SEG
6227
6228
6229	/**
6230	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6231	* context.
6232	*
6233	* @returns VBox status code.
6234	* @param pVCpu Pointer to the VMCPU.
6235	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6236	* out-of-sync. Make sure to update the required fields
6237	* before using them.
6238	*
6239	* @remarks No-long-jump zone!!!
6240	*/
6241	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6242	{
6243	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6244	{
6245	if (!pVCpu->hm.s.fUsingHyperDR7)
6246	{
6247	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6248	uint32_t u32Val;
6249	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6250	pMixedCtx->dr[7] = u32Val;
6251	}
6252
6253	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
6254	}
6255	return VINF_SUCCESS;
6256	}
6257
6258
6259	/**
6260	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6261	*
6262	* @returns VBox status code.
6263	* @param pVCpu Pointer to the VMCPU.
6264	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6265	* out-of-sync. Make sure to update the required fields
6266	* before using them.
6267	*
6268	* @remarks No-long-jump zone!!!
6269	*/
6270	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6271	{
6272	NOREF(pMixedCtx);
6273
6274	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6275	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
6276	return VINF_SUCCESS;
6277	}
6278
6279
6280	/**
6281	* Saves the entire guest state from the currently active VMCS into the
6282	* guest-CPU context. This essentially VMREADs all guest-data.
6283	*
6284	* @returns VBox status code.
6285	* @param pVCpu Pointer to the VMCPU.
6286	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6287	* out-of-sync. Make sure to update the required fields
6288	* before using them.
6289	*/
6290	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6291	{
6292	Assert(pVCpu);
6293	Assert(pMixedCtx);
6294
6295	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6296	return VINF_SUCCESS;
6297
6298	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6299	again on the ring-3 callback path, there is no real need to. */
6300	if (VMMRZCallRing3IsEnabled(pVCpu))
6301	VMMR0LogFlushDisable(pVCpu);
6302	else
6303	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6304	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6305
6306	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6308
6309	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6311
6312	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6320
6321	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6323
6324	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6326
6327	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6328	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6329
6330	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6331	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6332
6333	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6334	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6335
6336	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6337	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6338
6339	if (VMMRZCallRing3IsEnabled(pVCpu))
6340	VMMR0LogFlushEnable(pVCpu);
6341
6342	return rc;
6343	}
6344
6345
6346	/**
6347	* Check per-VM and per-VCPU force flag actions that require us to go back to
6348	* ring-3 for one reason or another.
6349	*
6350	* @returns VBox status code (information status code included).
6351	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6352	* ring-3.
6353	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6354	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6355	* interrupts)
6356	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6357	* all EMTs to be in ring-3.
6358	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6359	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6360	* to the EM loop.
6361	*
6362	* @param pVM Pointer to the VM.
6363	* @param pVCpu Pointer to the VMCPU.
6364	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6365	* out-of-sync. Make sure to update the required fields
6366	* before using them.
6367	*/
6368	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6369	{
6370	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6371
6372	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6373	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6374	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6375	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6376	{
6377	/* We need the control registers now, make sure the guest-CPU context is updated. */
6378	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6379	AssertRCReturn(rc3, rc3);
6380
6381	/* Pending HM CR3 sync. */
6382	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6383	{
6384	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6385	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6386	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6387	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6388	}
6389
6390	/* Pending HM PAE PDPEs. */
6391	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6392	{
6393	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6394	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6395	}
6396
6397	/* Pending PGM C3 sync. */
6398	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6399	{
6400	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6401	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6402	if (rc2 != VINF_SUCCESS)
6403	{
6404	AssertRC(rc2);
6405	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6406	return rc2;
6407	}
6408	}
6409
6410	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6411	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6412	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6413	{
6414	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6415	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6416	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6417	return rc2;
6418	}
6419
6420	/* Pending VM request packets, such as hardware interrupts. */
6421	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6422	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6423	{
6424	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6425	return VINF_EM_PENDING_REQUEST;
6426	}
6427
6428	/* Pending PGM pool flushes. */
6429	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6430	{
6431	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6432	return VINF_PGM_POOL_FLUSH_PENDING;
6433	}
6434
6435	/* Pending DMA requests. */
6436	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6437	{
6438	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6439	return VINF_EM_RAW_TO_R3;
6440	}
6441	}
6442
6443	return VINF_SUCCESS;
6444	}
6445
6446
6447	/**
6448	* Converts any TRPM trap into a pending HM event. This is typically used when
6449	* entering from ring-3 (not longjmp returns).
6450	*
6451	* @param pVCpu Pointer to the VMCPU.
6452	*/
6453	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6454	{
6455	Assert(TRPMHasTrap(pVCpu));
6456	Assert(!pVCpu->hm.s.Event.fPending);
6457
6458	uint8_t uVector;
6459	TRPMEVENT enmTrpmEvent;
6460	RTGCUINT uErrCode;
6461	RTGCUINTPTR GCPtrFaultAddress;
6462	uint8_t cbInstr;
6463
6464	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6465	AssertRC(rc);
6466
6467	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6468	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6469	if (enmTrpmEvent == TRPM_TRAP)
6470	{
6471	switch (uVector)
6472	{
6473	case X86_XCPT_BP:
6474	case X86_XCPT_OF:
6475	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6476	break;
6477
6478	case X86_XCPT_PF:
6479	case X86_XCPT_DF:
6480	case X86_XCPT_TS:
6481	case X86_XCPT_NP:
6482	case X86_XCPT_SS:
6483	case X86_XCPT_GP:
6484	case X86_XCPT_AC:
6485	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6486	/* no break! */
6487	default:
6488	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6489	break;
6490	}
6491	}
6492	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6493	{
6494	if (uVector == X86_XCPT_NMI)
6495	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6496	else
6497	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6498	}
6499	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6500	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6501	else
6502	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6503
6504	rc = TRPMResetTrap(pVCpu);
6505	AssertRC(rc);
6506	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6507	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6508
6509	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6510	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6511	}
6512
6513
6514	/**
6515	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6516	* VT-x to execute any instruction.
6517	*
6518	* @param pvCpu Pointer to the VMCPU.
6519	*/
6520	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6521	{
6522	Assert(pVCpu->hm.s.Event.fPending);
6523
6524	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6525	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6526	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6527	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6528
6529	/* If a trap was already pending, we did something wrong! */
6530	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6531
6532	TRPMEVENT enmTrapType;
6533	switch (uVectorType)
6534	{
6535	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6536	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6537	enmTrapType = TRPM_HARDWARE_INT;
6538	break;
6539
6540	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6541	enmTrapType = TRPM_SOFTWARE_INT;
6542	break;
6543
6544	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6545	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6546	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6547	enmTrapType = TRPM_TRAP;
6548	break;
6549
6550	default:
6551	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6552	enmTrapType = TRPM_32BIT_HACK;
6553	break;
6554	}
6555
6556	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6557
6558	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6559	AssertRC(rc);
6560
6561	if (fErrorCodeValid)
6562	TRPMSetErrorCode(pVCpu, uErrorCode);
6563
6564	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6565	&& uVector == X86_XCPT_PF)
6566	{
6567	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6568	}
6569	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6570	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6571	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6572	{
6573	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6574	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6575	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6576	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6577	}
6578	pVCpu->hm.s.Event.fPending = false;
6579	}
6580
6581
6582	/**
6583	* Does the necessary state syncing before returning to ring-3 for any reason
6584	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6585	*
6586	* @returns VBox status code.
6587	* @param pVM Pointer to the VM.
6588	* @param pVCpu Pointer to the VMCPU.
6589	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6590	* be out-of-sync. Make sure to update the required
6591	* fields before using them.
6592	* @param fSaveGuestState Whether to save the guest state or not.
6593	*
6594	* @remarks If you modify code here, make sure to check whether
6595	* hmR0VmxCallRing3Callback() needs to be updated too!!!
6596	* @remarks No-long-jmp zone!!!
6597	*/
6598	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6599	{
6600	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6601	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6602
6603	RTCPUID idCpu = RTMpCpuId();
6604	Log4Func(("HostCpuId=%u\n", idCpu));
6605
6606	/* Save the guest state if necessary. */
6607	if ( fSaveGuestState
6608	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6609	{
6610	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6611	AssertRCReturn(rc, rc);
6612	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6613	}
6614
6615	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6616	if (CPUMIsGuestFPUStateActive(pVCpu))
6617	{
6618	/* We shouldn't reload CR0 without saving it first. */
6619	if (!fSaveGuestState)
6620	{
6621	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6622	AssertRCReturn(rc, rc);
6623	}
6624	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6625	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6626	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6627	}
6628
6629	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6630	#ifdef VBOX_STRICT
6631	if (CPUMIsHyperDebugStateActive(pVCpu))
6632	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6633	#endif
6634	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6635	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6636	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6637	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6638
6639	#if HC_ARCH_BITS == 64
6640	/* Restore host-state bits that VT-x only restores partially. */
6641	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6642	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6643	{
6644	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6645	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6646	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6647	}
6648	#endif
6649
6650	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6651	/* Restore the host MSRs as we're leaving VT-x context. */
6652	if ( HMVMX_IS_64BIT_HOST_MODE()
6653	&& pVM->hm.s.fAllow64BitGuests
6654	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6655	{
6656	/* We shouldn't reload the guest MSRs without saving it first. */
6657	if (!fSaveGuestState)
6658	{
6659	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6660	AssertRCReturn(rc, rc);
6661	}
6662	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS);
6663	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6664	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6665	}
6666	#endif
6667
6668	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6669	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6670	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6671	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6672	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6673	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6674	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6675	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6676
6677	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6678
6679	/** @todo This partially defeats the purpose of having preemption hooks.
6680	* The problem is, deregistering the hooks should be moved to a place that
6681	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6682	* context.
6683	*/
6684	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6685	{
6686	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6687	AssertRCReturn(rc, rc);
6688
6689	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6690	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6691	}
6692	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6693	NOREF(idCpu);
6694
6695	return VINF_SUCCESS;
6696	}
6697
6698
6699	/**
6700	* Leaves the VT-x session.
6701	*
6702	* @returns VBox status code.
6703	* @param pVM Pointer to the VM.
6704	* @param pVCpu Pointer to the VMCPU.
6705	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6706	* out-of-sync. Make sure to update the required fields
6707	* before using them.
6708	*
6709	* @remarks No-long-jmp zone!!!
6710	*/
6711	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6712	{
6713	HM_DISABLE_PREEMPT_IF_NEEDED();
6714	HMVMX_ASSERT_CPU_SAFE();
6715	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6716	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6717
6718	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6719	and done this from the VMXR0ThreadCtxCallback(). */
6720	if (!pVCpu->hm.s.fLeaveDone)
6721	{
6722	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6723	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6724	pVCpu->hm.s.fLeaveDone = true;
6725	}
6726
6727	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6728	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6729	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6730	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6731	VMMR0ThreadCtxHooksDeregister(pVCpu);
6732
6733	/* Leave HM context. This takes care of local init (term). */
6734	int rc = HMR0LeaveCpu(pVCpu);
6735
6736	HM_RESTORE_PREEMPT_IF_NEEDED();
6737
6738	return rc;
6739	}
6740
6741
6742	/**
6743	* Does the necessary state syncing before doing a longjmp to ring-3.
6744	*
6745	* @returns VBox status code.
6746	* @param pVM Pointer to the VM.
6747	* @param pVCpu Pointer to the VMCPU.
6748	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6749	* out-of-sync. Make sure to update the required fields
6750	* before using them.
6751	*
6752	* @remarks No-long-jmp zone!!!
6753	*/
6754	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6755	{
6756	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6757	}
6758
6759
6760	/**
6761	* Take necessary actions before going back to ring-3.
6762	*
6763	* An action requires us to go back to ring-3. This function does the necessary
6764	* steps before we can safely return to ring-3. This is not the same as longjmps
6765	* to ring-3, this is voluntary and prepares the guest so it may continue
6766	* executing outside HM (recompiler/IEM).
6767	*
6768	* @returns VBox status code.
6769	* @param pVM Pointer to the VM.
6770	* @param pVCpu Pointer to the VMCPU.
6771	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6772	* out-of-sync. Make sure to update the required fields
6773	* before using them.
6774	* @param rcExit The reason for exiting to ring-3. Can be
6775	* VINF_VMM_UNKNOWN_RING3_CALL.
6776	*/
6777	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6778	{
6779	Assert(pVM);
6780	Assert(pVCpu);
6781	Assert(pMixedCtx);
6782	HMVMX_ASSERT_PREEMPT_SAFE();
6783
6784	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6785	{
6786	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6787	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6788	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6789	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6790	}
6791
6792	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6793	VMMRZCallRing3Disable(pVCpu);
6794	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6795
6796	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6797	if (pVCpu->hm.s.Event.fPending)
6798	{
6799	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6800	Assert(!pVCpu->hm.s.Event.fPending);
6801	}
6802
6803	/* Save guest state and restore host state bits. */
6804	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6805	AssertRCReturn(rc, rc);
6806	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6807
6808	/* Sync recompiler state. */
6809	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6810	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6811	\| CPUM_CHANGED_LDTR
6812	\| CPUM_CHANGED_GDTR
6813	\| CPUM_CHANGED_IDTR
6814	\| CPUM_CHANGED_TR
6815	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6816	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6817	if ( pVM->hm.s.fNestedPaging
6818	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6819	{
6820	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6821	}
6822
6823	Assert(!pVCpu->hm.s.fClearTrapFlag);
6824
6825	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6826	if (rcExit != VINF_EM_RAW_INTERRUPT)
6827	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6828
6829	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6830
6831	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6832	VMMRZCallRing3RemoveNotification(pVCpu);
6833	VMMRZCallRing3Enable(pVCpu);
6834
6835	return rc;
6836	}
6837
6838
6839	/**
6840	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6841	* longjump to ring-3 and possibly get preempted.
6842	*
6843	* @returns VBox status code.
6844	* @param pVCpu Pointer to the VMCPU.
6845	* @param enmOperation The operation causing the ring-3 longjump.
6846	* @param pvUser Opaque pointer to the guest-CPU context. The data
6847	* may be out-of-sync. Make sure to update the required
6848	* fields before using them.
6849	*
6850	* @remarks If you modify code here, make sure to check whether
6851	* hmR0VmxLeave() needs to be updated too!!!
6852	*/
6853	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6854	{
6855	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6856	{
6857	/* If anything here asserts or fails, good luck. */
6858	VMMRZCallRing3RemoveNotification(pVCpu);
6859	VMMRZCallRing3Disable(pVCpu);
6860	HM_DISABLE_PREEMPT_IF_NEEDED();
6861
6862	PVM pVM = pVCpu->CTX_SUFF(pVM);
6863	if (CPUMIsGuestFPUStateActive(pVCpu))
6864	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6865
6866	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6867
6868	#if HC_ARCH_BITS == 64
6869	/* Restore host-state bits that VT-x only restores partially. */
6870	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6871	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6872	{
6873	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6874	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6875	}
6876	#endif
6877
6878	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6879	/* Restore the host MSRs as we're leaving VT-x context. */
6880	if ( HMVMX_IS_64BIT_HOST_MODE()
6881	&& pVM->hm.s.fAllow64BitGuests
6882	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6883	{
6884	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6885	}
6886	#endif
6887	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6888	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6889	{
6890	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6891	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6892	}
6893
6894	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6895	VMMR0ThreadCtxHooksDeregister(pVCpu);
6896
6897	HMR0LeaveCpu(pVCpu);
6898	HM_RESTORE_PREEMPT_IF_NEEDED();
6899	return VINF_SUCCESS;
6900	}
6901
6902	Assert(pVCpu);
6903	Assert(pvUser);
6904	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6905	HMVMX_ASSERT_PREEMPT_SAFE();
6906
6907	VMMRZCallRing3Disable(pVCpu);
6908	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6909
6910	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6911	enmOperation));
6912
6913	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6914	AssertRCReturn(rc, rc);
6915
6916	VMMRZCallRing3Enable(pVCpu);
6917	return VINF_SUCCESS;
6918	}
6919
6920
6921	/**
6922	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6923	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6924	*
6925	* @param pVCpu Pointer to the VMCPU.
6926	*/
6927	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6928	{
6929	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6930	{
6931	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6932	{
6933	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6934	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6935	AssertRC(rc);
6936	}
6937	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6938	}
6939
6940
6941	/**
6942	* Evaluates the event to be delivered to the guest and sets it as the pending
6943	* event.
6944	*
6945	* @param pVCpu Pointer to the VMCPU.
6946	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6947	* out-of-sync. Make sure to update the required fields
6948	* before using them.
6949	*/
6950	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6951	{
6952	Assert(!pVCpu->hm.s.Event.fPending);
6953
6954	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6955	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6956	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6957	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6958
6959	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6960	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6961	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6962	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6963	Assert(!TRPMHasTrap(pVCpu));
6964
6965	/** @todo SMI. SMIs take priority over NMIs. */
6966	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6967	{
6968	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6969	if ( !fBlockMovSS
6970	&& !fBlockSti)
6971	{
6972	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6973	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6974	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6975	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6976
6977	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6978	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6979	}
6980	else
6981	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6982	}
6983	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6984	&& !pVCpu->hm.s.fSingleInstruction)
6985	{
6986	/*
6987	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6988	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6989	* evaluated here and not set as pending, solely based on the force-flags.
6990	*/
6991	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6992	AssertRC(rc);
6993	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6994	if ( !fBlockInt
6995	&& !fBlockSti
6996	&& !fBlockMovSS)
6997	{
6998	uint8_t u8Interrupt;
6999	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7000	if (RT_SUCCESS(rc))
7001	{
7002	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7003	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7004	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7005
7006	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7007	}
7008	else
7009	{
7010	/** @todo Does this actually happen? If not turn it into an assertion. */
7011	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7012	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7013	}
7014	}
7015	else
7016	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7017	}
7018	}
7019
7020
7021	/**
7022	* Injects any pending events into the guest if the guest is in a state to
7023	* receive them.
7024	*
7025	* @returns VBox status code (informational status codes included).
7026	* @param pVCpu Pointer to the VMCPU.
7027	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7028	* out-of-sync. Make sure to update the required fields
7029	* before using them.
7030	*/
7031	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7032	{
7033	HMVMX_ASSERT_PREEMPT_SAFE();
7034	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7035
7036	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7037	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7038	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7039	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7040
7041	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
7042	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7043	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7044	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7045	Assert(!TRPMHasTrap(pVCpu));
7046
7047	int rc = VINF_SUCCESS;
7048	if (pVCpu->hm.s.Event.fPending)
7049	{
7050	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7051	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7052	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7053	{
7054	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7055	AssertRCReturn(rc, rc);
7056	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7057	Assert(!fBlockInt);
7058	Assert(!fBlockSti);
7059	Assert(!fBlockMovSS);
7060	}
7061	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7062	{
7063	Assert(!fBlockSti);
7064	Assert(!fBlockMovSS);
7065	}
7066	#endif
7067	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7068	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7069	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7070	AssertRCReturn(rc, rc);
7071
7072	/* Update the interruptibility-state as it could have been changed by
7073	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7074	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7075	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7076
7077	#ifdef VBOX_WITH_STATISTICS
7078	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7079	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7080	else
7081	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7082	#endif
7083	}
7084
7085	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7086	if ( !pVCpu->hm.s.fSingleInstruction
7087	&& !DBGFIsStepping(pVCpu))
7088	{
7089	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7090	AssertRCReturn(rc2, rc2);
7091	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7092	{
7093	/*
7094	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7095	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7096	* See Intel spec. 27.3.4 "Saving Non-Register State".
7097	*/
7098	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7099	AssertRCReturn(rc2, rc2);
7100	}
7101	}
7102	else
7103	{
7104	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
7105	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
7106	uIntrState = 0;
7107	}
7108
7109	/*
7110	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7111	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7112	*/
7113	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7114	AssertRC(rc2);
7115
7116	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7117	NOREF(fBlockMovSS); NOREF(fBlockSti);
7118	return rc;
7119	}
7120
7121
7122	/**
7123	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7124	*
7125	* @param pVCpu Pointer to the VMCPU.
7126	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7127	* out-of-sync. Make sure to update the required fields
7128	* before using them.
7129	*/
7130	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7131	{
7132	NOREF(pMixedCtx);
7133	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7134	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7135	}
7136
7137
7138	/**
7139	* Injects a double-fault (#DF) exception into the VM.
7140	*
7141	* @returns VBox status code (informational status code included).
7142	* @param pVCpu Pointer to the VMCPU.
7143	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7144	* out-of-sync. Make sure to update the required fields
7145	* before using them.
7146	*/
7147	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7148	{
7149	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7150	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7151	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7152	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7153	puIntrState);
7154	}
7155
7156
7157	/**
7158	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7159	*
7160	* @param pVCpu Pointer to the VMCPU.
7161	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7162	* out-of-sync. Make sure to update the required fields
7163	* before using them.
7164	*/
7165	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7166	{
7167	NOREF(pMixedCtx);
7168	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7169	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7170	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7171	}
7172
7173
7174	/**
7175	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7176	*
7177	* @param pVCpu Pointer to the VMCPU.
7178	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7179	* out-of-sync. Make sure to update the required fields
7180	* before using them.
7181	* @param cbInstr The value of RIP that is to be pushed on the guest
7182	* stack.
7183	*/
7184	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7185	{
7186	NOREF(pMixedCtx);
7187	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7188	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7189	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7190	}
7191
7192
7193	/**
7194	* Injects a general-protection (#GP) fault into the VM.
7195	*
7196	* @returns VBox status code (informational status code included).
7197	* @param pVCpu Pointer to the VMCPU.
7198	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7199	* out-of-sync. Make sure to update the required fields
7200	* before using them.
7201	* @param u32ErrorCode The error code associated with the #GP.
7202	*/
7203	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7204	uint32_t *puIntrState)
7205	{
7206	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7207	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7208	if (fErrorCodeValid)
7209	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7210	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7211	puIntrState);
7212	}
7213
7214
7215	/**
7216	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7217	*
7218	* @param pVCpu Pointer to the VMCPU.
7219	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7220	* out-of-sync. Make sure to update the required fields
7221	* before using them.
7222	* @param uVector The software interrupt vector number.
7223	* @param cbInstr The value of RIP that is to be pushed on the guest
7224	* stack.
7225	*/
7226	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7227	{
7228	NOREF(pMixedCtx);
7229	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7230	if ( uVector == X86_XCPT_BP
7231	\|\| uVector == X86_XCPT_OF)
7232	{
7233	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7234	}
7235	else
7236	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7237	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7238	}
7239
7240
7241	/**
7242	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7243	* stack.
7244	*
7245	* @returns VBox status code (information status code included).
7246	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7247	* @param pVM Pointer to the VM.
7248	* @param pMixedCtx Pointer to the guest-CPU context.
7249	* @param uValue The value to push to the guest stack.
7250	*/
7251	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7252	{
7253	/*
7254	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7255	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7256	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7257	*/
7258	if (pMixedCtx->sp == 1)
7259	return VINF_EM_RESET;
7260	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7261	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7262	AssertRCReturn(rc, rc);
7263	return rc;
7264	}
7265
7266
7267	/**
7268	* Injects an event into the guest upon VM-entry by updating the relevant fields
7269	* in the VM-entry area in the VMCS.
7270	*
7271	* @returns VBox status code (informational error codes included).
7272	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7273	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7274	*
7275	* @param pVCpu Pointer to the VMCPU.
7276	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7277	* be out-of-sync. Make sure to update the required
7278	* fields before using them.
7279	* @param u64IntInfo The VM-entry interruption-information field.
7280	* @param cbInstr The VM-entry instruction length in bytes (for
7281	* software interrupts, exceptions and privileged
7282	* software exceptions).
7283	* @param u32ErrCode The VM-entry exception error code.
7284	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7285	* @param puIntrState Pointer to the current guest interruptibility-state.
7286	* This interruptibility-state will be updated if
7287	* necessary. This cannot not be NULL.
7288	*
7289	* @remarks Requires CR0!
7290	* @remarks No-long-jump zone!!!
7291	*/
7292	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7293	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7294	{
7295	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7296	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7297	Assert(puIntrState);
7298	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7299
7300	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7301	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7302
7303	#ifdef VBOX_STRICT
7304	/* Validate the error-code-valid bit for hardware exceptions. */
7305	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7306	{
7307	switch (uVector)
7308	{
7309	case X86_XCPT_PF:
7310	case X86_XCPT_DF:
7311	case X86_XCPT_TS:
7312	case X86_XCPT_NP:
7313	case X86_XCPT_SS:
7314	case X86_XCPT_GP:
7315	case X86_XCPT_AC:
7316	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7317	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7318	/* fallthru */
7319	default:
7320	break;
7321	}
7322	}
7323	#endif
7324
7325	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7326	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7327	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7328
7329	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7330
7331	/* We require CR0 to check if the guest is in real-mode. */
7332	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7333	AssertRCReturn(rc, rc);
7334
7335	/*
7336	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7337	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7338	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7339	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7340	*/
7341	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7342	{
7343	PVM pVM = pVCpu->CTX_SUFF(pVM);
7344	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7345	{
7346	Assert(PDMVmmDevHeapIsEnabled(pVM));
7347	Assert(pVM->hm.s.vmx.pRealModeTSS);
7348
7349	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7350	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7351	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7352	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7353	AssertRCReturn(rc, rc);
7354	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7355
7356	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7357	const size_t cbIdtEntry = sizeof(X86IDTR16);
7358	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7359	{
7360	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7361	if (uVector == X86_XCPT_DF)
7362	return VINF_EM_RESET;
7363	else if (uVector == X86_XCPT_GP)
7364	{
7365	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7366	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7367	}
7368
7369	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7370	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7371	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7372	}
7373
7374	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7375	uint16_t uGuestIp = pMixedCtx->ip;
7376	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7377	{
7378	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7379	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7380	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7381	}
7382	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7383	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7384
7385	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7386	X86IDTR16 IdtEntry;
7387	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7388	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7389	AssertRCReturn(rc, rc);
7390
7391	/* Construct the stack frame for the interrupt/exception handler. */
7392	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7393	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7394	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7395	AssertRCReturn(rc, rc);
7396
7397	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7398	if (rc == VINF_SUCCESS)
7399	{
7400	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7401	pMixedCtx->rip = IdtEntry.offSel;
7402	pMixedCtx->cs.Sel = IdtEntry.uSel;
7403	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7404	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7405	&& uVector == X86_XCPT_PF)
7406	{
7407	pMixedCtx->cr2 = GCPtrFaultAddress;
7408	}
7409
7410	/* If any other guest-state bits are changed here, make sure to update
7411	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7412	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7413	\| HM_CHANGED_GUEST_RIP
7414	\| HM_CHANGED_GUEST_RFLAGS
7415	\| HM_CHANGED_GUEST_RSP);
7416
7417	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7418	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7419	{
7420	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7421	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7422	Log4(("Clearing inhibition due to STI.\n"));
7423	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7424	}
7425	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7426
7427	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7428	it, if we are returning to ring-3 before executing guest code. */
7429	pVCpu->hm.s.Event.fPending = false;
7430	}
7431	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7432	return rc;
7433	}
7434	else
7435	{
7436	/*
7437	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7438	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7439	*/
7440	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7441	}
7442	}
7443
7444	/* Validate. */
7445	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7446	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7447	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7448
7449	/* Inject. */
7450	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7451	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7452	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7453	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7454
7455	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7456	&& uVector == X86_XCPT_PF)
7457	{
7458	pMixedCtx->cr2 = GCPtrFaultAddress;
7459	}
7460
7461	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7462	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7463
7464	AssertRCReturn(rc, rc);
7465	return rc;
7466	}
7467
7468
7469	/**
7470	* Clears the interrupt-window exiting control in the VMCS and if necessary
7471	* clears the current event in the VMCS as well.
7472	*
7473	* @returns VBox status code.
7474	* @param pVCpu Pointer to the VMCPU.
7475	*
7476	* @remarks Use this function only to clear events that have not yet been
7477	* delivered to the guest but are injected in the VMCS!
7478	* @remarks No-long-jump zone!!!
7479	*/
7480	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7481	{
7482	int rc;
7483	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7484
7485	/* Clear interrupt-window exiting control. */
7486	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7487	{
7488	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7489	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7490	AssertRC(rc);
7491	}
7492
7493	if (!pVCpu->hm.s.Event.fPending)
7494	return;
7495
7496	#ifdef VBOX_STRICT
7497	uint32_t u32EntryInfo;
7498	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7499	AssertRC(rc);
7500	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7501	#endif
7502
7503	/* Clear the entry-interruption field (including the valid bit). */
7504	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7505	AssertRC(rc);
7506
7507	/* Clear the pending debug exception field. */
7508	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7509	AssertRC(rc);
7510	}
7511
7512
7513	/**
7514	* Enters the VT-x session.
7515	*
7516	* @returns VBox status code.
7517	* @param pVM Pointer to the VM.
7518	* @param pVCpu Pointer to the VMCPU.
7519	* @param pCpu Pointer to the CPU info struct.
7520	*/
7521	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7522	{
7523	AssertPtr(pVM);
7524	AssertPtr(pVCpu);
7525	Assert(pVM->hm.s.vmx.fSupported);
7526	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7527	NOREF(pCpu); NOREF(pVM);
7528
7529	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7530	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7531
7532	#ifdef VBOX_STRICT
7533	/* Make sure we're in VMX root mode. */
7534	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7535	if (!(u32HostCR4 & X86_CR4_VMXE))
7536	{
7537	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7538	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7539	}
7540	#endif
7541
7542	/*
7543	* Load the VCPU's VMCS as the current (and active) one.
7544	*/
7545	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7546	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7547	if (RT_FAILURE(rc))
7548	return rc;
7549
7550	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7551	pVCpu->hm.s.fLeaveDone = false;
7552	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7553
7554	return VINF_SUCCESS;
7555	}
7556
7557
7558	/**
7559	* The thread-context callback (only on platforms which support it).
7560	*
7561	* @param enmEvent The thread-context event.
7562	* @param pVCpu Pointer to the VMCPU.
7563	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7564	* @thread EMT(pVCpu)
7565	*/
7566	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7567	{
7568	NOREF(fGlobalInit);
7569
7570	switch (enmEvent)
7571	{
7572	case RTTHREADCTXEVENT_PREEMPTING:
7573	{
7574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7575	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7576	VMCPU_ASSERT_EMT(pVCpu);
7577
7578	PVM pVM = pVCpu->CTX_SUFF(pVM);
7579	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7580
7581	/* No longjmps (logger flushes, locks) in this fragile context. */
7582	VMMRZCallRing3Disable(pVCpu);
7583	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7584
7585	/*
7586	* Restore host-state (FPU, debug etc.)
7587	*/
7588	if (!pVCpu->hm.s.fLeaveDone)
7589	{
7590	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7591	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7592	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7593	pVCpu->hm.s.fLeaveDone = true;
7594	}
7595
7596	/* Leave HM context, takes care of local init (term). */
7597	int rc = HMR0LeaveCpu(pVCpu);
7598	AssertRC(rc); NOREF(rc);
7599
7600	/* Restore longjmp state. */
7601	VMMRZCallRing3Enable(pVCpu);
7602	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7603	break;
7604	}
7605
7606	case RTTHREADCTXEVENT_RESUMED:
7607	{
7608	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7609	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7610	VMCPU_ASSERT_EMT(pVCpu);
7611
7612	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7613	VMMRZCallRing3Disable(pVCpu);
7614	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7615
7616	/* Initialize the bare minimum state required for HM. This takes care of
7617	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7618	int rc = HMR0EnterCpu(pVCpu);
7619	AssertRC(rc);
7620	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7621
7622	/* Load the active VMCS as the current one. */
7623	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7624	{
7625	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7626	AssertRC(rc); NOREF(rc);
7627	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7628	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7629	}
7630	pVCpu->hm.s.fLeaveDone = false;
7631
7632	/* Restore longjmp state. */
7633	VMMRZCallRing3Enable(pVCpu);
7634	break;
7635	}
7636
7637	default:
7638	break;
7639	}
7640	}
7641
7642
7643	/**
7644	* Saves the host state in the VMCS host-state.
7645	* Sets up the VM-exit MSR-load area.
7646	*
7647	* The CPU state will be loaded from these fields on every successful VM-exit.
7648	*
7649	* @returns VBox status code.
7650	* @param pVM Pointer to the VM.
7651	* @param pVCpu Pointer to the VMCPU.
7652	*
7653	* @remarks No-long-jump zone!!!
7654	*/
7655	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7656	{
7657	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7658
7659	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7660	return VINF_SUCCESS;
7661
7662	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7663	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7664
7665	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7666	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7667
7668	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7669	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7670
7671	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7672	return rc;
7673	}
7674
7675
7676	/**
7677	* Saves the host state in the VMCS host-state.
7678	*
7679	* @returns VBox status code.
7680	* @param pVM Pointer to the VM.
7681	* @param pVCpu Pointer to the VMCPU.
7682	*
7683	* @remarks No-long-jump zone!!!
7684	*/
7685	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7686	{
7687	AssertPtr(pVM);
7688	AssertPtr(pVCpu);
7689
7690	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7691
7692	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7693	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7694	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7695	return hmR0VmxSaveHostState(pVM, pVCpu);
7696	}
7697
7698
7699	/**
7700	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7701	* loaded from these fields on every successful VM-entry.
7702	*
7703	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7704	* Sets up the VM-entry controls.
7705	* Sets up the appropriate VMX non-root function to execute guest code based on
7706	* the guest CPU mode.
7707	*
7708	* @returns VBox status code.
7709	* @param pVM Pointer to the VM.
7710	* @param pVCpu Pointer to the VMCPU.
7711	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7712	* out-of-sync. Make sure to update the required fields
7713	* before using them.
7714	*
7715	* @remarks No-long-jump zone!!!
7716	*/
7717	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7718	{
7719	AssertPtr(pVM);
7720	AssertPtr(pVCpu);
7721	AssertPtr(pMixedCtx);
7722	HMVMX_ASSERT_PREEMPT_SAFE();
7723
7724	#ifdef LOG_ENABLED
7725	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7726	* probably not initialized yet? Anyway this will do for now.
7727	*
7728	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7729	* interface and disable ring-3 calls when thread-context hooks are not
7730	* available. */
7731	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7732	VMMR0LogFlushDisable(pVCpu);
7733	#endif
7734
7735	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7736
7737	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7738
7739	/* Determine real-on-v86 mode. */
7740	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7741	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7742	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7743	{
7744	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7745	}
7746
7747	/*
7748	* Load the guest-state into the VMCS.
7749	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7750	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7751	*/
7752	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7753	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7754
7755	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7756	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7757	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7758
7759	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7760	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7761	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7762
7763	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7764	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7765
7766	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7767	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7768
7769	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7770	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7771	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7772
7773	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7774	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7775
7776	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7777	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7778
7779	/*
7780	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7781	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7782	*/
7783	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7784	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7785
7786	/* Clear any unused and reserved bits. */
7787	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7788
7789	#ifdef LOG_ENABLED
7790	/* Only reenable log-flushing if the caller has it enabled. */
7791	if (!fCallerDisabledLogFlush)
7792	VMMR0LogFlushEnable(pVCpu);
7793	#endif
7794
7795	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7796	return rc;
7797	}
7798
7799
7800	/**
7801	* Loads the state shared between the host and guest into the VMCS.
7802	*
7803	* @param pVM Pointer to the VM.
7804	* @param pVCpu Pointer to the VMCPU.
7805	* @param pCtx Pointer to the guest-CPU context.
7806	*
7807	* @remarks No-long-jump zone!!!
7808	*/
7809	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7810	{
7811	NOREF(pVM);
7812
7813	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7814	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7815
7816	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7817	{
7818	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7819	AssertRC(rc);
7820	}
7821
7822	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7823	{
7824	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7825	AssertRC(rc);
7826
7827	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7828	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7829	{
7830	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7831	AssertRC(rc);
7832	}
7833	}
7834
7835	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7836	{
7837	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7838	if ( HMVMX_IS_64BIT_HOST_MODE()
7839	&& pVM->hm.s.fAllow64BitGuests)
7840	{
7841	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7842	}
7843	#endif
7844	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7845	}
7846
7847	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7848	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7849	}
7850
7851
7852	/**
7853	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7854	*
7855	* @param pVM Pointer to the VM.
7856	* @param pVCpu Pointer to the VMCPU.
7857	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7858	* out-of-sync. Make sure to update the required fields
7859	* before using them.
7860	*/
7861	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7862	{
7863	HMVMX_ASSERT_PREEMPT_SAFE();
7864
7865	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7866	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7867	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7868	#endif
7869
7870	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7871	{
7872	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7873	AssertRC(rc);
7874	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7875	}
7876	else if (VMCPU_HMCF_VALUE(pVCpu))
7877	{
7878	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7879	AssertRC(rc);
7880	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7881	}
7882
7883	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7884	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7885	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7886	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7887
7888	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7889	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7890	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7891	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7892	#endif
7893	}
7894
7895
7896	/**
7897	* Does the preparations before executing guest code in VT-x.
7898	*
7899	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7900	* recompiler. We must be cautious what we do here regarding committing
7901	* guest-state information into the VMCS assuming we assuredly execute the
7902	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7903	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7904	* so that the recompiler can (and should) use them when it resumes guest
7905	* execution. Otherwise such operations must be done when we can no longer
7906	* exit to ring-3.
7907	*
7908	* @returns Strict VBox status code.
7909	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7910	* have been disabled.
7911	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7912	* double-fault into the guest.
7913	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7914	*
7915	* @param pVM Pointer to the VM.
7916	* @param pVCpu Pointer to the VMCPU.
7917	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7918	* out-of-sync. Make sure to update the required fields
7919	* before using them.
7920	* @param pVmxTransient Pointer to the VMX transient structure.
7921	*/
7922	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7923	{
7924	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7925
7926	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7927	PGMRZDynMapFlushAutoSet(pVCpu);
7928	#endif
7929
7930	/* Check force flag actions that might require us to go back to ring-3. */
7931	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7932	if (rc != VINF_SUCCESS)
7933	return rc;
7934
7935	#ifndef IEM_VERIFICATION_MODE_FULL
7936	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7937	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7938	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7939	{
7940	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7941	RTGCPHYS GCPhysApicBase;
7942	GCPhysApicBase = pMixedCtx->msrApicBase;
7943	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7944
7945	/* Unalias any existing mapping. */
7946	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7947	AssertRCReturn(rc, rc);
7948
7949	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7950	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7951	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7952	AssertRCReturn(rc, rc);
7953
7954	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7955	}
7956	#endif /* !IEM_VERIFICATION_MODE_FULL */
7957
7958	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7959	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7960
7961	/*
7962	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7963	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7964	*/
7965	if (TRPMHasTrap(pVCpu))
7966	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7967	else if (!pVCpu->hm.s.Event.fPending)
7968	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7969
7970	/*
7971	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7972	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7973	*/
7974	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7975	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7976	{
7977	Assert(rc == VINF_EM_RESET);
7978	return rc;
7979	}
7980
7981	/*
7982	* No longjmps to ring-3 from this point on!!!
7983	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7984	* This also disables flushing of the R0-logger instance (if any).
7985	*/
7986	VMMRZCallRing3Disable(pVCpu);
7987
7988	/*
7989	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7990	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7991	*
7992	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7993	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7994	*
7995	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7996	* executing guest code.
7997	*/
7998	pVmxTransient->uEflags = ASMIntDisableFlags();
7999	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8000	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8001	{
8002	hmR0VmxClearEventVmcs(pVCpu);
8003	ASMSetFlags(pVmxTransient->uEflags);
8004	VMMRZCallRing3Enable(pVCpu);
8005	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8006	return VINF_EM_RAW_TO_R3;
8007	}
8008	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8009	{
8010	hmR0VmxClearEventVmcs(pVCpu);
8011	ASMSetFlags(pVmxTransient->uEflags);
8012	VMMRZCallRing3Enable(pVCpu);
8013	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8014	return VINF_EM_RAW_INTERRUPT;
8015	}
8016
8017	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8018	pVCpu->hm.s.Event.fPending = false;
8019
8020	return VINF_SUCCESS;
8021	}
8022
8023
8024	/**
8025	* Prepares to run guest code in VT-x and we've committed to doing so. This
8026	* means there is no backing out to ring-3 or anywhere else at this
8027	* point.
8028	*
8029	* @param pVM Pointer to the VM.
8030	* @param pVCpu Pointer to the VMCPU.
8031	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8032	* out-of-sync. Make sure to update the required fields
8033	* before using them.
8034	* @param pVmxTransient Pointer to the VMX transient structure.
8035	*
8036	* @remarks Called with preemption disabled.
8037	* @remarks No-long-jump zone!!!
8038	*/
8039	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8040	{
8041	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8042	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8043	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8044
8045	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8046	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8047
8048	/*
8049	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8050	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8051	* Reload only the necessary state, the assertion will catch if other parts of the code
8052	* change.
8053	*/
8054	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8055	{
8056	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8057	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8058	}
8059
8060	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8061	if (!CPUMIsGuestFPUStateActive(pVCpu))
8062	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8063	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8064	#endif
8065
8066	if ( pVCpu->hm.s.fUseGuestFpu
8067	&& !CPUMIsGuestFPUStateActive(pVCpu))
8068	{
8069	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8070	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
8071	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8072	}
8073
8074	/*
8075	* The host MSR values the very first time around won't be updated, so we need to
8076	* fill those values in. Subsequently, it's updated as part of the host state.
8077	*/
8078	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8079	&& pVCpu->hm.s.vmx.cMsrs > 0)
8080	{
8081	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8082	}
8083
8084	/*
8085	* Load the host state bits as we may've been preempted (only happens when
8086	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8087	*/
8088	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8089	{
8090	/* This ASSUMES that pfnStartVM has been set up already. */
8091	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8092	AssertRC(rc);
8093	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8094	}
8095	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8096
8097	/*
8098	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8099	*/
8100	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8101	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8102	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
8103
8104	/* Store status of the shared guest-host state at the time of VM-entry. */
8105	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8106	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8107	{
8108	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8109	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8110	}
8111	else
8112	#endif
8113	{
8114	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8115	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8116	}
8117	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8118
8119	/*
8120	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8121	*/
8122	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8123	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8124
8125	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8126	RTCPUID idCurrentCpu = pCpu->idCpu;
8127	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8128	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8129	{
8130	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8131	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8132	}
8133
8134	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8135	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8136	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8137	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8138
8139	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8140
8141	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8142	to start executing. */
8143
8144	/*
8145	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8146	*/
8147	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8148	{
8149	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8150	{
8151	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8152	AssertRC(rc2);
8153	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
8154	uint6_t u64GuestTscAuxMsr;
8155	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTscAuxMsr);
8156	AssertRC(rc2);
8157	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, u64GuestTscAuxMsr, true /* fUpdateHostMsr */);
8158	}
8159	else
8160	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8161	}
8162	#ifdef VBOX_STRICT
8163	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8164	#endif
8165	}
8166
8167
8168	/**
8169	* Performs some essential restoration of state after running guest code in
8170	* VT-x.
8171	*
8172	* @param pVM Pointer to the VM.
8173	* @param pVCpu Pointer to the VMCPU.
8174	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8175	* out-of-sync. Make sure to update the required fields
8176	* before using them.
8177	* @param pVmxTransient Pointer to the VMX transient structure.
8178	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8179	*
8180	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8181	*
8182	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8183	* unconditionally when it is safe to do so.
8184	*/
8185	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8186	{
8187	NOREF(pVM);
8188
8189	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8190
8191	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8192	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8193	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8194	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8195	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8196
8197	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8198	{
8199	/** @todo Find a way to fix hardcoding a guestimate. */
8200	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8201	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8202	}
8203
8204	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8205	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8206	Assert(!(ASMGetFlags() & X86_EFL_IF));
8207	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8208
8209	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8210	if (CPUMIsGuestFPUStateActive(pVCpu))
8211	{
8212	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8213	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8214	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8215	}
8216	#endif
8217
8218	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8219	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8220	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8221	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8222
8223	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8224	uint32_t uExitReason;
8225	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8226	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8227	AssertRC(rc);
8228	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8229	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8230
8231	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8232	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8233	{
8234	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8235	pVmxTransient->fVMEntryFailed));
8236	return;
8237	}
8238
8239	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8240	{
8241	/* Update the guest interruptibility-state from the VMCS. */
8242	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8243	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8244	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8245	AssertRC(rc);
8246	#endif
8247	/*
8248	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8249	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8250	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8251	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8252	*/
8253	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8254	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8255	{
8256	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8257	AssertRC(rc);
8258	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8259	}
8260	}
8261	}
8262
8263
8264
8265	/**
8266	* Runs the guest code using VT-x the normal way.
8267	*
8268	* @returns VBox status code.
8269	* @param pVM Pointer to the VM.
8270	* @param pVCpu Pointer to the VMCPU.
8271	* @param pCtx Pointer to the guest-CPU context.
8272	*
8273	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8274	*/
8275	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8276	{
8277	VMXTRANSIENT VmxTransient;
8278	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8279	int rc = VERR_INTERNAL_ERROR_5;
8280	uint32_t cLoops = 0;
8281
8282	for (;; cLoops++)
8283	{
8284	Assert(!HMR0SuspendPending());
8285	HMVMX_ASSERT_CPU_SAFE();
8286
8287	/* Preparatory work for running guest code, this may force us to return
8288	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8289	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8290	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8291	if (rc != VINF_SUCCESS)
8292	break;
8293
8294	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8295	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8296	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8297
8298	/* Restore any residual host-state and save any bits shared between host
8299	and guest into the guest-CPU state. Re-enables interrupts! */
8300	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8301
8302	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8303	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8304	{
8305	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8306	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8307	return rc;
8308	}
8309
8310	/* Handle the VM-exit. */
8311	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8312	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8313	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8314	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8315	HMVMX_START_EXIT_DISPATCH_PROF();
8316	#ifdef HMVMX_USE_FUNCTION_TABLE
8317	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8318	#else
8319	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8320	#endif
8321	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8322	if (rc != VINF_SUCCESS)
8323	break;
8324	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8325	{
8326	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8327	rc = VINF_EM_RAW_INTERRUPT;
8328	break;
8329	}
8330	}
8331
8332	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8333	return rc;
8334	}
8335
8336
8337	/**
8338	* Single steps guest code using VT-x.
8339	*
8340	* @returns VBox status code.
8341	* @param pVM Pointer to the VM.
8342	* @param pVCpu Pointer to the VMCPU.
8343	* @param pCtx Pointer to the guest-CPU context.
8344	*
8345	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8346	*/
8347	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8348	{
8349	VMXTRANSIENT VmxTransient;
8350	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8351	int rc = VERR_INTERNAL_ERROR_5;
8352	uint32_t cLoops = 0;
8353	uint16_t uCsStart = pCtx->cs.Sel;
8354	uint64_t uRipStart = pCtx->rip;
8355
8356	for (;; cLoops++)
8357	{
8358	Assert(!HMR0SuspendPending());
8359	HMVMX_ASSERT_CPU_SAFE();
8360
8361	/* Preparatory work for running guest code, this may force us to return
8362	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8363	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8364	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8365	if (rc != VINF_SUCCESS)
8366	break;
8367
8368	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8369	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8370	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8371
8372	/* Restore any residual host-state and save any bits shared between host
8373	and guest into the guest-CPU state. Re-enables interrupts! */
8374	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8375
8376	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8377	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8378	{
8379	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8380	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8381	return rc;
8382	}
8383
8384	/* Handle the VM-exit. */
8385	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8386	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8387	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8388	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8389	HMVMX_START_EXIT_DISPATCH_PROF();
8390	#ifdef HMVMX_USE_FUNCTION_TABLE
8391	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8392	#else
8393	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8394	#endif
8395	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8396	if (rc != VINF_SUCCESS)
8397	break;
8398	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8399	{
8400	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8401	rc = VINF_EM_RAW_INTERRUPT;
8402	break;
8403	}
8404
8405	/*
8406	* Did the RIP change, if so, consider it a single step.
8407	* Otherwise, make sure one of the TFs gets set.
8408	*/
8409	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8410	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8411	AssertRCReturn(rc2, rc2);
8412	if ( pCtx->rip != uRipStart
8413	\|\| pCtx->cs.Sel != uCsStart)
8414	{
8415	rc = VINF_EM_DBG_STEPPED;
8416	break;
8417	}
8418	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8419	}
8420
8421	/*
8422	* Clear the X86_EFL_TF if necessary.
8423	*/
8424	if (pVCpu->hm.s.fClearTrapFlag)
8425	{
8426	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8427	AssertRCReturn(rc2, rc2);
8428	pVCpu->hm.s.fClearTrapFlag = false;
8429	pCtx->eflags.Bits.u1TF = 0;
8430	}
8431	/** @todo there seems to be issues with the resume flag when the monitor trap
8432	* flag is pending without being used. Seen early in bios init when
8433	* accessing APIC page in prot mode. */
8434
8435	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8436	return rc;
8437	}
8438
8439
8440	/**
8441	* Runs the guest code using VT-x.
8442	*
8443	* @returns VBox status code.
8444	* @param pVM Pointer to the VM.
8445	* @param pVCpu Pointer to the VMCPU.
8446	* @param pCtx Pointer to the guest-CPU context.
8447	*/
8448	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8449	{
8450	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8451	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8452	HMVMX_ASSERT_PREEMPT_SAFE();
8453
8454	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8455
8456	int rc;
8457	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8458	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8459	else
8460	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8461
8462	if (rc == VERR_EM_INTERPRETER)
8463	rc = VINF_EM_RAW_EMULATE_INSTR;
8464	else if (rc == VINF_EM_RESET)
8465	rc = VINF_EM_TRIPLE_FAULT;
8466
8467	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8468	if (RT_FAILURE(rc2))
8469	{
8470	pVCpu->hm.s.u32HMError = rc;
8471	rc = rc2;
8472	}
8473	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8474	return rc;
8475	}
8476
8477
8478	#ifndef HMVMX_USE_FUNCTION_TABLE
8479	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8480	{
8481	#ifdef DEBUG_ramshankar
8482	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8483	# define LDVMCS() do { VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8484	#endif
8485	int rc;
8486	switch (rcReason)
8487	{
8488	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8489	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8490	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8491	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8492	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8493	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8494	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8495	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8496	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8497	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8498	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8499	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8500	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8508	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8509	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8510	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8511	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8512	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8513	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8514	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8515	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8516	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8517	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8518	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8519	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8520	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8521
8522	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8523	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8524	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8525	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8526	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8527	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8528	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8529	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8530	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8531
8532	case VMX_EXIT_VMCALL:
8533	case VMX_EXIT_VMCLEAR:
8534	case VMX_EXIT_VMLAUNCH:
8535	case VMX_EXIT_VMPTRLD:
8536	case VMX_EXIT_VMPTRST:
8537	case VMX_EXIT_VMREAD:
8538	case VMX_EXIT_VMRESUME:
8539	case VMX_EXIT_VMWRITE:
8540	case VMX_EXIT_VMXOFF:
8541	case VMX_EXIT_VMXON:
8542	case VMX_EXIT_INVEPT:
8543	case VMX_EXIT_INVVPID:
8544	case VMX_EXIT_VMFUNC:
8545	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8546	break;
8547	default:
8548	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8549	break;
8550	}
8551	return rc;
8552	}
8553	#endif
8554
8555	#ifdef DEBUG
8556	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8557	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8558	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8559
8560	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8561	do \
8562	{ \
8563	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8564	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8565	} while (0)
8566
8567	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8568	do { \
8569	AssertPtr(pVCpu); \
8570	AssertPtr(pMixedCtx); \
8571	AssertPtr(pVmxTransient); \
8572	Assert(pVmxTransient->fVMEntryFailed == false); \
8573	Assert(ASMIntAreEnabled()); \
8574	HMVMX_ASSERT_PREEMPT_SAFE(); \
8575	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8576	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8577	HMVMX_ASSERT_PREEMPT_SAFE(); \
8578	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8579	HMVMX_ASSERT_PREEMPT_CPUID(); \
8580	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8581	} while (0)
8582
8583	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8584	do { \
8585	Log4Func(("\n")); \
8586	} while (0)
8587	#else /* Release builds */
8588	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8589	do { \
8590	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8591	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8592	} while (0)
8593	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8594	#endif
8595
8596
8597	/**
8598	* Advances the guest RIP after reading it from the VMCS.
8599	*
8600	* @returns VBox status code.
8601	* @param pVCpu Pointer to the VMCPU.
8602	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8603	* out-of-sync. Make sure to update the required fields
8604	* before using them.
8605	* @param pVmxTransient Pointer to the VMX transient structure.
8606	*
8607	* @remarks No-long-jump zone!!!
8608	*/
8609	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8610	{
8611	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8612	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8613	AssertRCReturn(rc, rc);
8614
8615	pMixedCtx->rip += pVmxTransient->cbInstr;
8616	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8617	return rc;
8618	}
8619
8620
8621	/**
8622	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8623	* and update error record fields accordingly.
8624	*
8625	* @return VMX_IGS_* return codes.
8626	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8627	* wrong with the guest state.
8628	*
8629	* @param pVM Pointer to the VM.
8630	* @param pVCpu Pointer to the VMCPU.
8631	* @param pCtx Pointer to the guest-CPU state.
8632	*/
8633	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8634	{
8635	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8636	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8637	uError = (err); \
8638	break; \
8639	} else do { } while (0)
8640	/* Duplicate of IEM_IS_CANONICAL(). */
8641	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8642
8643	int rc;
8644	uint32_t uError = VMX_IGS_ERROR;
8645	uint32_t u32Val;
8646	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8647
8648	do
8649	{
8650	/*
8651	* CR0.
8652	*/
8653	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8654	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8655	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8656	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8657	if (fUnrestrictedGuest)
8658	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8659
8660	uint32_t u32GuestCR0;
8661	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8662	AssertRCBreak(rc);
8663	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8664	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8665	if ( !fUnrestrictedGuest
8666	&& (u32GuestCR0 & X86_CR0_PG)
8667	&& !(u32GuestCR0 & X86_CR0_PE))
8668	{
8669	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8670	}
8671
8672	/*
8673	* CR4.
8674	*/
8675	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8676	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8677
8678	uint32_t u32GuestCR4;
8679	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8680	AssertRCBreak(rc);
8681	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8682	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8683
8684	/*
8685	* IA32_DEBUGCTL MSR.
8686	*/
8687	uint64_t u64Val;
8688	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8689	AssertRCBreak(rc);
8690	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8691	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8692	{
8693	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8694	}
8695	uint64_t u64DebugCtlMsr = u64Val;
8696
8697	#ifdef VBOX_STRICT
8698	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8699	AssertRCBreak(rc);
8700	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8701	#endif
8702	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8703
8704	/*
8705	* RIP and RFLAGS.
8706	*/
8707	uint32_t u32Eflags;
8708	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8709	if (HMVMX_IS_64BIT_HOST_MODE())
8710	{
8711	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8712	AssertRCBreak(rc);
8713	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8714	if ( !fLongModeGuest
8715	\|\| !pCtx->cs.Attr.n.u1Long)
8716	{
8717	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8718	}
8719	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8720	* must be identical if the "IA32e mode guest" VM-entry control is 1
8721	* and CS.L is 1. No check applies if the CPU supports 64
8722	* linear-address bits. */
8723
8724	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8725	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8726	AssertRCBreak(rc);
8727	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8728	VMX_IGS_RFLAGS_RESERVED);
8729	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8730	u32Eflags = u64Val;
8731	}
8732	else
8733	#endif
8734	{
8735	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8736	AssertRCBreak(rc);
8737	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8738	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8739	}
8740
8741	if ( fLongModeGuest
8742	\|\| ( fUnrestrictedGuest
8743	&& !(u32GuestCR0 & X86_CR0_PE)))
8744	{
8745	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8746	}
8747
8748	uint32_t u32EntryInfo;
8749	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8750	AssertRCBreak(rc);
8751	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8752	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8753	{
8754	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8755	}
8756
8757	/*
8758	* 64-bit checks.
8759	*/
8760	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8761	if (HMVMX_IS_64BIT_HOST_MODE())
8762	{
8763	if ( fLongModeGuest
8764	&& !fUnrestrictedGuest)
8765	{
8766	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8767	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8768	}
8769
8770	if ( !fLongModeGuest
8771	&& (u32GuestCR4 & X86_CR4_PCIDE))
8772	{
8773	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8774	}
8775
8776	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8777	* 51:32 beyond the processor's physical-address width are 0. */
8778
8779	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8780	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8781	{
8782	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8783	}
8784
8785	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8786	AssertRCBreak(rc);
8787	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8788
8789	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8790	AssertRCBreak(rc);
8791	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8792	}
8793	#endif
8794
8795	/*
8796	* PERF_GLOBAL MSR.
8797	*/
8798	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8799	{
8800	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8801	AssertRCBreak(rc);
8802	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8803	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8804	}
8805
8806	/*
8807	* PAT MSR.
8808	*/
8809	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8810	{
8811	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8812	AssertRCBreak(rc);
8813	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8814	for (unsigned i = 0; i < 8; i++)
8815	{
8816	uint8_t u8Val = (u64Val & 0x7);
8817	if ( u8Val != 0 /* UC */
8818	\|\| u8Val != 1 /* WC */
8819	\|\| u8Val != 4 /* WT */
8820	\|\| u8Val != 5 /* WP */
8821	\|\| u8Val != 6 /* WB */
8822	\|\| u8Val != 7 /* UC- */)
8823	{
8824	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8825	}
8826	u64Val >>= 3;
8827	}
8828	}
8829
8830	/*
8831	* EFER MSR.
8832	*/
8833	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8834	{
8835	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8836	AssertRCBreak(rc);
8837	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8838	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8839	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8840	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8841	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8842	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8843	}
8844
8845	/*
8846	* Segment registers.
8847	*/
8848	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8849	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8850	if (!(u32Eflags & X86_EFL_VM))
8851	{
8852	/* CS */
8853	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8854	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8855	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8856	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8857	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8858	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8859	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8860	/* CS cannot be loaded with NULL in protected mode. */
8861	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8862	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8863	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8864	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8865	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8866	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8867	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8868	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8869	else
8870	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8871
8872	/* SS */
8873	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8874	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8875	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8876	if ( !(pCtx->cr0 & X86_CR0_PE)
8877	\|\| pCtx->cs.Attr.n.u4Type == 3)
8878	{
8879	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8880	}
8881	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8882	{
8883	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8884	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8885	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8886	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8887	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8888	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8889	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8890	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8891	}
8892
8893	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8894	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8895	{
8896	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8897	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8898	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8899	\|\| pCtx->ds.Attr.n.u4Type > 11
8900	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8901	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8902	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8903	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8904	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8905	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8906	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8907	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8908	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8909	}
8910	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8911	{
8912	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8913	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8914	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8915	\|\| pCtx->es.Attr.n.u4Type > 11
8916	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8917	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8918	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8919	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8920	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8921	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8922	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8923	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8924	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8925	}
8926	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8927	{
8928	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8929	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8930	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8931	\|\| pCtx->fs.Attr.n.u4Type > 11
8932	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8933	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8934	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8935	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8936	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8937	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8938	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8939	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8940	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8941	}
8942	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8943	{
8944	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8945	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8946	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8947	\|\| pCtx->gs.Attr.n.u4Type > 11
8948	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8949	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8950	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8951	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8952	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8953	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8954	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8955	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8956	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8957	}
8958	/* 64-bit capable CPUs. */
8959	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8960	if (HMVMX_IS_64BIT_HOST_MODE())
8961	{
8962	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8963	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8964	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8965	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8966	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8967	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8968	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8969	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8970	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8971	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8972	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8973	}
8974	#endif
8975	}
8976	else
8977	{
8978	/* V86 mode checks. */
8979	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8980	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8981	{
8982	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8983	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8984	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8985	}
8986	else
8987	{
8988	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8989	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8990	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8991	}
8992
8993	/* CS */
8994	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8995	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8996	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8997	/* SS */
8998	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8999	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9000	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9001	/* DS */
9002	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9003	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9004	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9005	/* ES */
9006	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9007	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9008	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9009	/* FS */
9010	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9011	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9012	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9013	/* GS */
9014	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9015	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9016	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9017	/* 64-bit capable CPUs. */
9018	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9019	if (HMVMX_IS_64BIT_HOST_MODE())
9020	{
9021	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9022	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9023	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9024	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9025	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9026	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9027	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9028	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9029	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9030	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9031	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9032	}
9033	#endif
9034	}
9035
9036	/*
9037	* TR.
9038	*/
9039	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9040	/* 64-bit capable CPUs. */
9041	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9042	if (HMVMX_IS_64BIT_HOST_MODE())
9043	{
9044	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9045	}
9046	#endif
9047	if (fLongModeGuest)
9048	{
9049	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9050	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9051	}
9052	else
9053	{
9054	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9055	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9056	VMX_IGS_TR_ATTR_TYPE_INVALID);
9057	}
9058	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9059	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9060	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9061	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9062	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9063	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9064	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9065	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9066
9067	/*
9068	* GDTR and IDTR.
9069	*/
9070	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9071	if (HMVMX_IS_64BIT_HOST_MODE())
9072	{
9073	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9074	AssertRCBreak(rc);
9075	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9076
9077	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9078	AssertRCBreak(rc);
9079	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9080	}
9081	#endif
9082
9083	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9084	AssertRCBreak(rc);
9085	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9086
9087	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9088	AssertRCBreak(rc);
9089	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9090
9091	/*
9092	* Guest Non-Register State.
9093	*/
9094	/* Activity State. */
9095	uint32_t u32ActivityState;
9096	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9097	AssertRCBreak(rc);
9098	HMVMX_CHECK_BREAK( !u32ActivityState
9099	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9100	VMX_IGS_ACTIVITY_STATE_INVALID);
9101	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9102	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9103	uint32_t u32IntrState;
9104	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9105	AssertRCBreak(rc);
9106	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9107	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9108	{
9109	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9110	}
9111
9112	/** @todo Activity state and injecting interrupts. Left as a todo since we
9113	* currently don't use activity states but ACTIVE. */
9114
9115	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9116	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9117
9118	/* Guest interruptibility-state. */
9119	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9120	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9121	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9122	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9123	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9124	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9125	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9126	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9127	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9128	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9129	{
9130	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9131	{
9132	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9133	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9134	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9135	}
9136	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9137	{
9138	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9139	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9140	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9141	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9142	}
9143	}
9144	/** @todo Assumes the processor is not in SMM. */
9145	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9146	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9147	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9148	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9149	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9150	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9151	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9152	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9153	{
9154	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9155	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9156	}
9157
9158	/* Pending debug exceptions. */
9159	if (HMVMX_IS_64BIT_HOST_MODE())
9160	{
9161	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9162	AssertRCBreak(rc);
9163	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9164	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9165	u32Val = u64Val; /* For pending debug exceptions checks below. */
9166	}
9167	else
9168	{
9169	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9170	AssertRCBreak(rc);
9171	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9172	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9173	}
9174
9175	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9176	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9177	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9178	{
9179	if ( (u32Eflags & X86_EFL_TF)
9180	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9181	{
9182	/* Bit 14 is PendingDebug.BS. */
9183	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9184	}
9185	if ( !(u32Eflags & X86_EFL_TF)
9186	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9187	{
9188	/* Bit 14 is PendingDebug.BS. */
9189	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9190	}
9191	}
9192
9193	/* VMCS link pointer. */
9194	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9195	AssertRCBreak(rc);
9196	if (u64Val != UINT64_C(0xffffffffffffffff))
9197	{
9198	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9199	/** @todo Bits beyond the processor's physical-address width MBZ. */
9200	/** @todo 32-bit located in memory referenced by value of this field (as a
9201	* physical address) must contain the processor's VMCS revision ID. */
9202	/** @todo SMM checks. */
9203	}
9204
9205	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
9206
9207	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9208	if (uError == VMX_IGS_ERROR)
9209	uError = VMX_IGS_REASON_NOT_FOUND;
9210	} while (0);
9211
9212	pVCpu->hm.s.u32HMError = uError;
9213	return uError;
9214
9215	#undef HMVMX_ERROR_BREAK
9216	#undef HMVMX_CHECK_BREAK
9217	#undef HMVMX_IS_CANONICAL
9218	}
9219
9220	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9221	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9222	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9223
9224	/** @name VM-exit handlers.
9225	* @{
9226	*/
9227
9228	/**
9229	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9230	*/
9231	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9232	{
9233	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9235	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9236	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9237	return VINF_SUCCESS;
9238	return VINF_EM_RAW_INTERRUPT;
9239	}
9240
9241
9242	/**
9243	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9244	*/
9245	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9246	{
9247	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9248	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9249
9250	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9251	AssertRCReturn(rc, rc);
9252
9253	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9254	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9255	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9256	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9257
9258	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9259	{
9260	/*
9261	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9262	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9263	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9264	*
9265	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9266	*/
9267	VMXDispatchHostNmi();
9268	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9269	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9270	return VINF_SUCCESS;
9271	}
9272
9273	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9274	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9275	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9276	{
9277	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9278	return VINF_SUCCESS;
9279	}
9280	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9281	{
9282	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9283	return rc;
9284	}
9285
9286	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9287	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9288	switch (uIntType)
9289	{
9290	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9291	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9292	/* no break */
9293	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9294	{
9295	switch (uVector)
9296	{
9297	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9298	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9299	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9300	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9301	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9302	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9303	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9304	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9305	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9306	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9307	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9308	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9309	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9310	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9311	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9312	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9313	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9314	#endif
9315	default:
9316	{
9317	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9318	AssertRCReturn(rc, rc);
9319
9320	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9321	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9322	{
9323	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9324	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9325	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9326
9327	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9328	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9329	AssertRCReturn(rc, rc);
9330	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9331	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9332	0 /* GCPtrFaultAddress */);
9333	AssertRCReturn(rc, rc);
9334	}
9335	else
9336	{
9337	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9338	pVCpu->hm.s.u32HMError = uVector;
9339	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9340	}
9341	break;
9342	}
9343	}
9344	break;
9345	}
9346
9347	default:
9348	{
9349	pVCpu->hm.s.u32HMError = uExitIntInfo;
9350	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9351	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9352	break;
9353	}
9354	}
9355	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9356	return rc;
9357	}
9358
9359
9360	/**
9361	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9362	*/
9363	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9364	{
9365	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9366
9367	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9368	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9369	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9370	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9371	AssertRCReturn(rc, rc);
9372
9373	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9374	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9375	return VINF_SUCCESS;
9376	}
9377
9378
9379	/**
9380	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9381	*/
9382	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9383	{
9384	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9385	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9386	HMVMX_RETURN_UNEXPECTED_EXIT();
9387	}
9388
9389
9390	/**
9391	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9392	*/
9393	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9394	{
9395	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9397	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9398	}
9399
9400
9401	/**
9402	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9403	*/
9404	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9405	{
9406	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9407	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9408	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9409	}
9410
9411
9412	/**
9413	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9414	*/
9415	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9416	{
9417	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9418	PVM pVM = pVCpu->CTX_SUFF(pVM);
9419	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9420	if (RT_LIKELY(rc == VINF_SUCCESS))
9421	{
9422	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9423	Assert(pVmxTransient->cbInstr == 2);
9424	}
9425	else
9426	{
9427	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9428	rc = VERR_EM_INTERPRETER;
9429	}
9430	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9431	return rc;
9432	}
9433
9434
9435	/**
9436	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9437	*/
9438	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9439	{
9440	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9441	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9442	AssertRCReturn(rc, rc);
9443
9444	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9445	return VINF_EM_RAW_EMULATE_INSTR;
9446
9447	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9448	HMVMX_RETURN_UNEXPECTED_EXIT();
9449	}
9450
9451
9452	/**
9453	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9454	*/
9455	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9456	{
9457	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9458	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9459	AssertRCReturn(rc, rc);
9460
9461	PVM pVM = pVCpu->CTX_SUFF(pVM);
9462	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9463	if (RT_LIKELY(rc == VINF_SUCCESS))
9464	{
9465	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9466	Assert(pVmxTransient->cbInstr == 2);
9467	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9468	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9469	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9470	}
9471	else
9472	{
9473	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9474	rc = VERR_EM_INTERPRETER;
9475	}
9476	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9477	return rc;
9478	}
9479
9480
9481	/**
9482	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9483	*/
9484	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9485	{
9486	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9487	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9488	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9489	AssertRCReturn(rc, rc);
9490
9491	PVM pVM = pVCpu->CTX_SUFF(pVM);
9492	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9493	if (RT_LIKELY(rc == VINF_SUCCESS))
9494	{
9495	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9496	Assert(pVmxTransient->cbInstr == 3);
9497	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9498	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9499	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9500	}
9501	else
9502	{
9503	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9504	rc = VERR_EM_INTERPRETER;
9505	}
9506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9507	return rc;
9508	}
9509
9510
9511	/**
9512	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9513	*/
9514	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9515	{
9516	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9517	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9518	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9519	AssertRCReturn(rc, rc);
9520
9521	PVM pVM = pVCpu->CTX_SUFF(pVM);
9522	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9523	if (RT_LIKELY(rc == VINF_SUCCESS))
9524	{
9525	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9526	Assert(pVmxTransient->cbInstr == 2);
9527	}
9528	else
9529	{
9530	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9531	rc = VERR_EM_INTERPRETER;
9532	}
9533	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9534	return rc;
9535	}
9536
9537
9538	/**
9539	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9540	*/
9541	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9542	{
9543	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9544	PVM pVM = pVCpu->CTX_SUFF(pVM);
9545	Assert(!pVM->hm.s.fNestedPaging);
9546
9547	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9548	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9549	AssertRCReturn(rc, rc);
9550
9551	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9552	rc = VBOXSTRICTRC_VAL(rc2);
9553	if (RT_LIKELY(rc == VINF_SUCCESS))
9554	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9555	else
9556	{
9557	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9558	pVmxTransient->uExitQualification, rc));
9559	}
9560	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9561	return rc;
9562	}
9563
9564
9565	/**
9566	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9567	*/
9568	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9569	{
9570	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9571	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9572	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9573	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9574	AssertRCReturn(rc, rc);
9575
9576	PVM pVM = pVCpu->CTX_SUFF(pVM);
9577	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9578	if (RT_LIKELY(rc == VINF_SUCCESS))
9579	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9580	else
9581	{
9582	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9583	rc = VERR_EM_INTERPRETER;
9584	}
9585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9586	return rc;
9587	}
9588
9589
9590	/**
9591	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9592	*/
9593	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9594	{
9595	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9596	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9597	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9598	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9599	AssertRCReturn(rc, rc);
9600
9601	PVM pVM = pVCpu->CTX_SUFF(pVM);
9602	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9603	rc = VBOXSTRICTRC_VAL(rc2);
9604	if (RT_LIKELY( rc == VINF_SUCCESS
9605	\|\| rc == VINF_EM_HALT))
9606	{
9607	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9608	AssertRCReturn(rc3, rc3);
9609
9610	if ( rc == VINF_EM_HALT
9611	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9612	{
9613	rc = VINF_SUCCESS;
9614	}
9615	}
9616	else
9617	{
9618	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9619	rc = VERR_EM_INTERPRETER;
9620	}
9621	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9622	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9623	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9624	return rc;
9625	}
9626
9627
9628	/**
9629	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9630	*/
9631	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9632	{
9633	/*
9634	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9635	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9636	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9637	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9638	*/
9639	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9640	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9641	HMVMX_RETURN_UNEXPECTED_EXIT();
9642	}
9643
9644
9645	/**
9646	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9647	*/
9648	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9649	{
9650	/*
9651	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9652	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9653	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9654	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9655	*/
9656	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9657	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9658	HMVMX_RETURN_UNEXPECTED_EXIT();
9659	}
9660
9661
9662	/**
9663	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9664	*/
9665	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9666	{
9667	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9668	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9669	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9670	HMVMX_RETURN_UNEXPECTED_EXIT();
9671	}
9672
9673
9674	/**
9675	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9676	*/
9677	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9678	{
9679	/*
9680	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9681	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9682	* See Intel spec. 25.3 "Other Causes of VM-exits".
9683	*/
9684	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9685	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9686	HMVMX_RETURN_UNEXPECTED_EXIT();
9687	}
9688
9689
9690	/**
9691	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9692	* VM-exit.
9693	*/
9694	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9695	{
9696	/*
9697	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9698	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9699	*
9700	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9701	* See Intel spec. "23.8 Restrictions on VMX operation".
9702	*/
9703	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9704	return VINF_SUCCESS;
9705	}
9706
9707
9708	/**
9709	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9710	* VM-exit.
9711	*/
9712	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9713	{
9714	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9715	return VINF_EM_RESET;
9716	}
9717
9718
9719	/**
9720	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9721	*/
9722	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9723	{
9724	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9725	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9726	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9727	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9728	AssertRCReturn(rc, rc);
9729
9730	pMixedCtx->rip++;
9731	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9732	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9733	rc = VINF_SUCCESS;
9734	else
9735	rc = VINF_EM_HALT;
9736
9737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9738	return rc;
9739	}
9740
9741
9742	/**
9743	* VM-exit handler for instructions that result in a #UD exception delivered to
9744	* the guest.
9745	*/
9746	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9747	{
9748	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9749	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9750	return VINF_SUCCESS;
9751	}
9752
9753
9754	/**
9755	* VM-exit handler for expiry of the VMX preemption timer.
9756	*/
9757	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9758	{
9759	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9760
9761	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9762	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9763
9764	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9765	PVM pVM = pVCpu->CTX_SUFF(pVM);
9766	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9767	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9768	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9769	}
9770
9771
9772	/**
9773	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9774	*/
9775	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9776	{
9777	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9778
9779	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9780	/** @todo check if XSETBV is supported by the recompiler. */
9781	return VERR_EM_INTERPRETER;
9782	}
9783
9784
9785	/**
9786	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9787	*/
9788	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9789	{
9790	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9791
9792	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9793	/** @todo implement EMInterpretInvpcid() */
9794	return VERR_EM_INTERPRETER;
9795	}
9796
9797
9798	/**
9799	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9800	* Error VM-exit.
9801	*/
9802	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9803	{
9804	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9805	AssertRCReturn(rc, rc);
9806
9807	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9808	NOREF(uInvalidReason);
9809
9810	#ifdef VBOX_STRICT
9811	uint32_t uIntrState;
9812	HMVMXHCUINTREG uHCReg;
9813	uint64_t u64Val;
9814	uint32_t u32Val;
9815
9816	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9817	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9818	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9819	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9820	AssertRCReturn(rc, rc);
9821
9822	Log4(("uInvalidReason %u\n", uInvalidReason));
9823	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9824	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9825	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9826	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9827
9828	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9829	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9830	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9831	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9832	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9833	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9834	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9835	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9836	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9837	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9838	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9839	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9840	#else
9841	NOREF(pVmxTransient);
9842	#endif
9843
9844	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9845	return VERR_VMX_INVALID_GUEST_STATE;
9846	}
9847
9848
9849	/**
9850	* VM-exit handler for VM-entry failure due to an MSR-load
9851	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9852	*/
9853	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9854	{
9855	NOREF(pVmxTransient);
9856	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9857	HMVMX_RETURN_UNEXPECTED_EXIT();
9858	}
9859
9860
9861	/**
9862	* VM-exit handler for VM-entry failure due to a machine-check event
9863	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9864	*/
9865	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9866	{
9867	NOREF(pVmxTransient);
9868	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9869	HMVMX_RETURN_UNEXPECTED_EXIT();
9870	}
9871
9872
9873	/**
9874	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9875	* theory.
9876	*/
9877	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9878	{
9879	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9880	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9881	return VERR_VMX_UNDEFINED_EXIT_CODE;
9882	}
9883
9884
9885	/**
9886	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9887	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9888	* Conditional VM-exit.
9889	*/
9890	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9891	{
9892	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9893
9894	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9896	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9897	return VERR_EM_INTERPRETER;
9898	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9899	HMVMX_RETURN_UNEXPECTED_EXIT();
9900	}
9901
9902
9903	/**
9904	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9905	*/
9906	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9907	{
9908	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9909
9910	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9912	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9913	return VERR_EM_INTERPRETER;
9914	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9915	HMVMX_RETURN_UNEXPECTED_EXIT();
9916	}
9917
9918
9919	/**
9920	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9921	*/
9922	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9923	{
9924	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9925
9926	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9927	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9928	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9929	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9930	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9931	{
9932	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9933	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9934	}
9935	AssertRCReturn(rc, rc);
9936	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9937
9938	#ifdef VBOX_STRICT
9939	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9940	{
9941	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9942	{
9943	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9944	HMVMX_RETURN_UNEXPECTED_EXIT();
9945	}
9946	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9947	if ( HMVMX_IS_64BIT_HOST_MODE()
9948	&& pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
9949	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
9950	{
9951	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
9952	HMVMX_RETURN_UNEXPECTED_EXIT();
9953	}
9954	# endif
9955	}
9956	#endif
9957
9958	PVM pVM = pVCpu->CTX_SUFF(pVM);
9959	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9960	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9961	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9962	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9963
9964	if (RT_LIKELY(rc == VINF_SUCCESS))
9965	{
9966	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9967	Assert(pVmxTransient->cbInstr == 2);
9968	}
9969	return rc;
9970	}
9971
9972
9973	/**
9974	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9975	*/
9976	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9977	{
9978	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9979	PVM pVM = pVCpu->CTX_SUFF(pVM);
9980	int rc = VINF_SUCCESS;
9981
9982	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9983	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9984	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9985	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9986	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9987	{
9988	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9989	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9990	}
9991	AssertRCReturn(rc, rc);
9992	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9993
9994	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9995	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9996	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9997
9998	if (RT_LIKELY(rc == VINF_SUCCESS))
9999	{
10000	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10001
10002	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10003	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10004	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10005	{
10006	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10007	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10008	EMInterpretWrmsr() changes it. */
10009	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10010	}
10011	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10012	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10013
10014	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10015	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10016	{
10017	switch (pMixedCtx->ecx)
10018	{
10019	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10020	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10021	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10022	case MSR_K8_FS_BASE: /* no break */
10023	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10024	default:
10025	{
10026	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10027	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10028	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10029	else if ( HMVMX_IS_64BIT_HOST_MODE()
10030	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10031	{
10032	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10033	}
10034	#endif
10035	break;
10036	}
10037	}
10038	}
10039	#ifdef VBOX_STRICT
10040	else
10041	{
10042	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10043	switch (pMixedCtx->ecx)
10044	{
10045	case MSR_IA32_SYSENTER_CS:
10046	case MSR_IA32_SYSENTER_EIP:
10047	case MSR_IA32_SYSENTER_ESP:
10048	case MSR_K8_FS_BASE:
10049	case MSR_K8_GS_BASE:
10050	{
10051	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10052	HMVMX_RETURN_UNEXPECTED_EXIT();
10053	}
10054
10055	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10056	default:
10057	{
10058	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10059	{
10060	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10061	pMixedCtx->ecx));
10062	HMVMX_RETURN_UNEXPECTED_EXIT();
10063	}
10064
10065	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10066	if ( HMVMX_IS_64BIT_HOST_MODE()
10067	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10068	{
10069	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10070	HMVMX_RETURN_UNEXPECTED_EXIT();
10071	}
10072	#endif
10073	break;
10074	}
10075	}
10076	}
10077	#endif /* VBOX_STRICT */
10078	}
10079	return rc;
10080	}
10081
10082
10083	/**
10084	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10085	*/
10086	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10087	{
10088	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10089
10090	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10091	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10092	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10093	return VERR_EM_INTERPRETER;
10094	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10095	HMVMX_RETURN_UNEXPECTED_EXIT();
10096	}
10097
10098
10099	/**
10100	* VM-exit handler for when the TPR value is lowered below the specified
10101	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10102	*/
10103	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10104	{
10105	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10106	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10107
10108	/*
10109	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10110	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10111	* resume guest execution.
10112	*/
10113	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10114	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10115	return VINF_SUCCESS;
10116	}
10117
10118
10119	/**
10120	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10121	* VM-exit.
10122	*
10123	* @retval VINF_SUCCESS when guest execution can continue.
10124	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10125	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10126	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10127	* recompiler.
10128	*/
10129	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10130	{
10131	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10132	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10133	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10134	AssertRCReturn(rc, rc);
10135
10136	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10137	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10138	PVM pVM = pVCpu->CTX_SUFF(pVM);
10139	switch (uAccessType)
10140	{
10141	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10142	{
10143	#if 0
10144	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10145	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10146	#else
10147	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10148	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10149	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10150	#endif
10151	AssertRCReturn(rc, rc);
10152
10153	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10154	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10155	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10156	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10157
10158	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10159	{
10160	case 0: /* CR0 */
10161	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10162	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10163	break;
10164	case 2: /* CR2 */
10165	/* Nothing to do here, CR2 it's not part of the VMCS. */
10166	break;
10167	case 3: /* CR3 */
10168	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10169	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10170	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10171	break;
10172	case 4: /* CR4 */
10173	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10174	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10175	break;
10176	case 8: /* CR8 */
10177	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10178	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10179	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10180	break;
10181	default:
10182	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10183	break;
10184	}
10185
10186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10187	break;
10188	}
10189
10190	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10191	{
10192	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10193	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10194	AssertRCReturn(rc, rc);
10195	Assert( !pVM->hm.s.fNestedPaging
10196	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10197	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10198
10199	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10200	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10201	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10202
10203	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10204	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10205	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10206	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10207	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10208	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10209	break;
10210	}
10211
10212	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10213	{
10214	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10215	AssertRCReturn(rc, rc);
10216	rc = EMInterpretCLTS(pVM, pVCpu);
10217	AssertRCReturn(rc, rc);
10218	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10219	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10220	Log4(("CRX CLTS write rc=%d\n", rc));
10221	break;
10222	}
10223
10224	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10225	{
10226	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10227	AssertRCReturn(rc, rc);
10228	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10229	if (RT_LIKELY(rc == VINF_SUCCESS))
10230	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10232	Log4(("CRX LMSW write rc=%d\n", rc));
10233	break;
10234	}
10235
10236	default:
10237	{
10238	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10239	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10240	}
10241	}
10242
10243	/* Validate possible error codes. */
10244	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10245	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10246	if (RT_SUCCESS(rc))
10247	{
10248	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10249	AssertRCReturn(rc2, rc2);
10250	}
10251
10252	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10253	return rc;
10254	}
10255
10256
10257	/**
10258	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10259	* VM-exit.
10260	*/
10261	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10262	{
10263	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10264	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10265
10266	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10267	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10268	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10269	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10270	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10271	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10272	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10273	AssertRCReturn(rc2, rc2);
10274
10275	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10276	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10277	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10278	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10279	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10280	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10281	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10282
10283	/* I/O operation lookup arrays. */
10284	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10285	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10286
10287	VBOXSTRICTRC rcStrict;
10288	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10289	const uint32_t cbInstr = pVmxTransient->cbInstr;
10290	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10291	PVM pVM = pVCpu->CTX_SUFF(pVM);
10292	if (fIOString)
10293	{
10294	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10295	/*
10296	* INS/OUTS - I/O String instruction.
10297	*
10298	* Use instruction-information if available, otherwise fall back on
10299	* interpreting the instruction.
10300	*/
10301	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10302	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10303	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10304	{
10305	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10306	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10307	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10308	AssertRCReturn(rc2, rc2);
10309	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10310	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10311	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10312	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10313	if (fIOWrite)
10314	{
10315	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10316	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10317	}
10318	else
10319	{
10320	/*
10321	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10322	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10323	* See Intel Instruction spec. for "INS".
10324	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10325	*/
10326	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10327	}
10328	}
10329	else
10330	{
10331	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10332	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10333	AssertRCReturn(rc2, rc2);
10334	rcStrict = IEMExecOne(pVCpu);
10335	}
10336	/** @todo IEM needs to be setting these flags somehow. */
10337	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10338	fUpdateRipAlready = true;
10339	#else
10340	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10341	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10342	if (RT_SUCCESS(rcStrict))
10343	{
10344	if (fIOWrite)
10345	{
10346	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10347	(DISCPUMODE)pDis->uAddrMode, cbValue);
10348	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10349	}
10350	else
10351	{
10352	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10353	(DISCPUMODE)pDis->uAddrMode, cbValue);
10354	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10355	}
10356	}
10357	else
10358	{
10359	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10360	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10361	}
10362	#endif
10363	}
10364	else
10365	{
10366	/*
10367	* IN/OUT - I/O instruction.
10368	*/
10369	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10370	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10371	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10372	if (fIOWrite)
10373	{
10374	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10375	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10376	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10377	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10378	}
10379	else
10380	{
10381	uint32_t u32Result = 0;
10382	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10383	if (IOM_SUCCESS(rcStrict))
10384	{
10385	/* Save result of I/O IN instr. in AL/AX/EAX. */
10386	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10387	}
10388	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10389	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10390	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10391	}
10392	}
10393
10394	if (IOM_SUCCESS(rcStrict))
10395	{
10396	if (!fUpdateRipAlready)
10397	{
10398	pMixedCtx->rip += cbInstr;
10399	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10400	}
10401
10402	/*
10403	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10404	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10405	*/
10406	if (fIOString)
10407	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10408
10409	/*
10410	* If any I/O breakpoints are armed, we need to check if one triggered
10411	* and take appropriate action.
10412	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10413	*/
10414	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10415	AssertRCReturn(rc2, rc2);
10416
10417	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10418	* execution engines about whether hyper BPs and such are pending. */
10419	uint32_t const uDr7 = pMixedCtx->dr[7];
10420	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10421	&& X86_DR7_ANY_RW_IO(uDr7)
10422	&& (pMixedCtx->cr4 & X86_CR4_DE))
10423	\|\| DBGFBpIsHwIoArmed(pVM)))
10424	{
10425	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10426
10427	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10428	VMMRZCallRing3Disable(pVCpu);
10429	HM_DISABLE_PREEMPT_IF_NEEDED();
10430
10431	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10432
10433	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10434	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10435	{
10436	/* Raise #DB. */
10437	if (fIsGuestDbgActive)
10438	ASMSetDR6(pMixedCtx->dr[6]);
10439	if (pMixedCtx->dr[7] != uDr7)
10440	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10441
10442	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10443	}
10444	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10445	else if ( rcStrict2 != VINF_SUCCESS
10446	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10447	rcStrict = rcStrict2;
10448
10449	HM_RESTORE_PREEMPT_IF_NEEDED();
10450	VMMRZCallRing3Enable(pVCpu);
10451	}
10452	}
10453
10454	#ifdef DEBUG
10455	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10456	Assert(!fIOWrite);
10457	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10458	Assert(fIOWrite);
10459	else
10460	{
10461	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10462	* statuses, that the VMM device and some others may return. See
10463	* IOM_SUCCESS() for guidance. */
10464	AssertMsg( RT_FAILURE(rcStrict)
10465	\|\| rcStrict == VINF_SUCCESS
10466	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10467	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10468	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10469	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10470	}
10471	#endif
10472
10473	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10474	return VBOXSTRICTRC_TODO(rcStrict);
10475	}
10476
10477
10478	/**
10479	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10480	* VM-exit.
10481	*/
10482	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10483	{
10484	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10485
10486	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10487	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10488	AssertRCReturn(rc, rc);
10489	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10490	{
10491	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10492	AssertRCReturn(rc, rc);
10493	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10494	{
10495	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10496
10497	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10498	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10499	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10500	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10501	{
10502	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10503	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10504
10505	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10506	Assert(!pVCpu->hm.s.Event.fPending);
10507	pVCpu->hm.s.Event.fPending = true;
10508	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10509	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10510	AssertRCReturn(rc, rc);
10511	if (fErrorCodeValid)
10512	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10513	else
10514	pVCpu->hm.s.Event.u32ErrCode = 0;
10515	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10516	&& uVector == X86_XCPT_PF)
10517	{
10518	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10519	}
10520
10521	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10522	}
10523	}
10524	}
10525
10526	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10527	* emulation. */
10528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10529	return VERR_EM_INTERPRETER;
10530	}
10531
10532
10533	/**
10534	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10535	*/
10536	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10537	{
10538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10539	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10540	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10541	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10542	AssertRCReturn(rc, rc);
10543	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10544	return VINF_EM_DBG_STEPPED;
10545	}
10546
10547
10548	/**
10549	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10550	*/
10551	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10552	{
10553	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10554
10555	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10556	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10557	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10558	return VINF_SUCCESS;
10559	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10560	return rc;
10561
10562	#if 0
10563	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10564	* just sync the whole thing. */
10565	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10566	#else
10567	/* Aggressive state sync. for now. */
10568	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10569	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10570	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10571	#endif
10572	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10573	AssertRCReturn(rc, rc);
10574
10575	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10576	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10577	switch (uAccessType)
10578	{
10579	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10580	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10581	{
10582	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10583	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10584	{
10585	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10586	}
10587
10588	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10589	GCPhys &= PAGE_BASE_GC_MASK;
10590	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10591	PVM pVM = pVCpu->CTX_SUFF(pVM);
10592	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10593	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10594
10595	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10596	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10597	CPUMCTX2CORE(pMixedCtx), GCPhys);
10598	rc = VBOXSTRICTRC_VAL(rc2);
10599	Log4(("ApicAccess rc=%d\n", rc));
10600	if ( rc == VINF_SUCCESS
10601	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10602	\|\| rc == VERR_PAGE_NOT_PRESENT)
10603	{
10604	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10605	\| HM_CHANGED_GUEST_RSP
10606	\| HM_CHANGED_GUEST_RFLAGS
10607	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10608	rc = VINF_SUCCESS;
10609	}
10610	break;
10611	}
10612
10613	default:
10614	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10615	rc = VINF_EM_RAW_EMULATE_INSTR;
10616	break;
10617	}
10618
10619	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10620	return rc;
10621	}
10622
10623
10624	/**
10625	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10626	* VM-exit.
10627	*/
10628	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10629	{
10630	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10631
10632	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10633	if (pVmxTransient->fWasGuestDebugStateActive)
10634	{
10635	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10636	HMVMX_RETURN_UNEXPECTED_EXIT();
10637	}
10638
10639	int rc = VERR_INTERNAL_ERROR_5;
10640	if ( !DBGFIsStepping(pVCpu)
10641	&& !pVCpu->hm.s.fSingleInstruction
10642	&& !pVmxTransient->fWasHyperDebugStateActive)
10643	{
10644	/* Don't intercept MOV DRx and #DB any more. */
10645	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10646	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10647	AssertRCReturn(rc, rc);
10648
10649	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10650	{
10651	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10652	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10653	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10654	AssertRCReturn(rc, rc);
10655	#endif
10656	}
10657
10658	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10659	VMMRZCallRing3Disable(pVCpu);
10660	HM_DISABLE_PREEMPT_IF_NEEDED();
10661
10662	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10663	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10664	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10665
10666	HM_RESTORE_PREEMPT_IF_NEEDED();
10667	VMMRZCallRing3Enable(pVCpu);
10668
10669	#ifdef VBOX_WITH_STATISTICS
10670	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10671	AssertRCReturn(rc, rc);
10672	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10674	else
10675	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10676	#endif
10677	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10678	return VINF_SUCCESS;
10679	}
10680
10681	/*
10682	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10683	* Update the segment registers and DR7 from the CPU.
10684	*/
10685	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10686	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10687	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10688	AssertRCReturn(rc, rc);
10689	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10690
10691	PVM pVM = pVCpu->CTX_SUFF(pVM);
10692	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10693	{
10694	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10695	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10696	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10697	if (RT_SUCCESS(rc))
10698	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10699	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10700	}
10701	else
10702	{
10703	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10704	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10705	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10706	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10707	}
10708
10709	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10710	if (RT_SUCCESS(rc))
10711	{
10712	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10713	AssertRCReturn(rc2, rc2);
10714	}
10715	return rc;
10716	}
10717
10718
10719	/**
10720	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10721	* Conditional VM-exit.
10722	*/
10723	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10724	{
10725	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10726	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10727
10728	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10729	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10730	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10731	return VINF_SUCCESS;
10732	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10733	return rc;
10734
10735	RTGCPHYS GCPhys = 0;
10736	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10737
10738	#if 0
10739	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10740	#else
10741	/* Aggressive state sync. for now. */
10742	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10743	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10744	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10745	#endif
10746	AssertRCReturn(rc, rc);
10747
10748	/*
10749	* If we succeed, resume guest execution.
10750	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10751	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10752	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10753	* weird case. See @bugref{6043}.
10754	*/
10755	PVM pVM = pVCpu->CTX_SUFF(pVM);
10756	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10757	rc = VBOXSTRICTRC_VAL(rc2);
10758	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10759	if ( rc == VINF_SUCCESS
10760	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10761	\|\| rc == VERR_PAGE_NOT_PRESENT)
10762	{
10763	/* Successfully handled MMIO operation. */
10764	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10765	\| HM_CHANGED_GUEST_RSP
10766	\| HM_CHANGED_GUEST_RFLAGS
10767	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10768	rc = VINF_SUCCESS;
10769	}
10770	return rc;
10771	}
10772
10773
10774	/**
10775	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10776	* VM-exit.
10777	*/
10778	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10779	{
10780	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10781	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10782
10783	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10784	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10785	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10786	return VINF_SUCCESS;
10787	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10788	return rc;
10789
10790	RTGCPHYS GCPhys = 0;
10791	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10792	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10793	#if 0
10794	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10795	#else
10796	/* Aggressive state sync. for now. */
10797	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10798	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10799	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10800	#endif
10801	AssertRCReturn(rc, rc);
10802
10803	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10804	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10805
10806	RTGCUINT uErrorCode = 0;
10807	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10808	uErrorCode \|= X86_TRAP_PF_ID;
10809	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10810	uErrorCode \|= X86_TRAP_PF_RW;
10811	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10812	uErrorCode \|= X86_TRAP_PF_P;
10813
10814	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10815
10816	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10817	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10818
10819	/* Handle the pagefault trap for the nested shadow table. */
10820	PVM pVM = pVCpu->CTX_SUFF(pVM);
10821	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10822	TRPMResetTrap(pVCpu);
10823
10824	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10825	if ( rc == VINF_SUCCESS
10826	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10827	\|\| rc == VERR_PAGE_NOT_PRESENT)
10828	{
10829	/* Successfully synced our nested page tables. */
10830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10831	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10832	\| HM_CHANGED_GUEST_RSP
10833	\| HM_CHANGED_GUEST_RFLAGS);
10834	return VINF_SUCCESS;
10835	}
10836
10837	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10838	return rc;
10839	}
10840
10841	/** @} */
10842
10843	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10844	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10845	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10846
10847	/** @name VM-exit exception handlers.
10848	* @{
10849	*/
10850
10851	/**
10852	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10853	*/
10854	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10855	{
10856	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10858
10859	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10860	AssertRCReturn(rc, rc);
10861
10862	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10863	{
10864	/* Old-style FPU error reporting needs some extra work. */
10865	/** @todo don't fall back to the recompiler, but do it manually. */
10866	return VERR_EM_INTERPRETER;
10867	}
10868
10869	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10870	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10871	return rc;
10872	}
10873
10874
10875	/**
10876	* VM-exit exception handler for #BP (Breakpoint exception).
10877	*/
10878	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10879	{
10880	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10881	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10882
10883	/** @todo Try optimize this by not saving the entire guest state unless
10884	* really needed. */
10885	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10886	AssertRCReturn(rc, rc);
10887
10888	PVM pVM = pVCpu->CTX_SUFF(pVM);
10889	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10890	if (rc == VINF_EM_RAW_GUEST_TRAP)
10891	{
10892	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10893	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10894	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10895	AssertRCReturn(rc, rc);
10896
10897	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10898	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10899	}
10900
10901	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10902	return rc;
10903	}
10904
10905
10906	/**
10907	* VM-exit exception handler for #DB (Debug exception).
10908	*/
10909	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10910	{
10911	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10912	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10913	Log6(("XcptDB\n"));
10914
10915	/*
10916	* Get the DR6-like values from the exit qualification and pass it to DBGF
10917	* for processing.
10918	*/
10919	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10920	AssertRCReturn(rc, rc);
10921
10922	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10923	uint64_t uDR6 = X86_DR6_INIT_VAL;
10924	uDR6 \|= ( pVmxTransient->uExitQualification
10925	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10926
10927	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10928	if (rc == VINF_EM_RAW_GUEST_TRAP)
10929	{
10930	/*
10931	* The exception was for the guest. Update DR6, DR7.GD and
10932	* IA32_DEBUGCTL.LBR before forwarding it.
10933	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10934	*/
10935	VMMRZCallRing3Disable(pVCpu);
10936	HM_DISABLE_PREEMPT_IF_NEEDED();
10937
10938	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10939	pMixedCtx->dr[6] \|= uDR6;
10940	if (CPUMIsGuestDebugStateActive(pVCpu))
10941	ASMSetDR6(pMixedCtx->dr[6]);
10942
10943	HM_RESTORE_PREEMPT_IF_NEEDED();
10944	VMMRZCallRing3Enable(pVCpu);
10945
10946	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10947	AssertRCReturn(rc, rc);
10948
10949	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10950	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10951
10952	/* Paranoia. */
10953	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10954	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10955
10956	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10957	AssertRCReturn(rc, rc);
10958
10959	/*
10960	* Raise #DB in the guest.
10961	*/
10962	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10963	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10964	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10965	AssertRCReturn(rc, rc);
10966	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10967	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10968	return VINF_SUCCESS;
10969	}
10970
10971	/*
10972	* Not a guest trap, must be a hypervisor related debug event then.
10973	* Update DR6 in case someone is interested in it.
10974	*/
10975	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10976	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10977	CPUMSetHyperDR6(pVCpu, uDR6);
10978
10979	return rc;
10980	}
10981
10982
10983	/**
10984	* VM-exit exception handler for #NM (Device-not-available exception: floating
10985	* point exception).
10986	*/
10987	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10988	{
10989	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10990
10991	/* We require CR0 and EFER. EFER is always up-to-date. */
10992	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10993	AssertRCReturn(rc, rc);
10994
10995	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10996	VMMRZCallRing3Disable(pVCpu);
10997	HM_DISABLE_PREEMPT_IF_NEEDED();
10998
10999	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11000	if (pVmxTransient->fWasGuestFPUStateActive)
11001	{
11002	rc = VINF_EM_RAW_GUEST_TRAP;
11003	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11004	}
11005	else
11006	{
11007	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11008	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11009	#endif
11010	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11011	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11012	}
11013
11014	HM_RESTORE_PREEMPT_IF_NEEDED();
11015	VMMRZCallRing3Enable(pVCpu);
11016
11017	if (rc == VINF_SUCCESS)
11018	{
11019	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11020	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11021	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11022	pVCpu->hm.s.fUseGuestFpu = true;
11023	}
11024	else
11025	{
11026	/* Forward #NM to the guest. */
11027	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11028	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11029	AssertRCReturn(rc, rc);
11030	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11031	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11032	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11033	}
11034
11035	return VINF_SUCCESS;
11036	}
11037
11038
11039	/**
11040	* VM-exit exception handler for #GP (General-protection exception).
11041	*
11042	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11043	*/
11044	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11045	{
11046	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11047	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11048
11049	int rc = VERR_INTERNAL_ERROR_5;
11050	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11051	{
11052	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11053	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11054	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11055	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11056	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11057	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11058	AssertRCReturn(rc, rc);
11059	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11060	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11061	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11062	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11063	return rc;
11064	#else
11065	/* We don't intercept #GP. */
11066	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11067	NOREF(pVmxTransient);
11068	return VERR_VMX_UNEXPECTED_EXCEPTION;
11069	#endif
11070	}
11071
11072	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11073	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11074
11075	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11076	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11077	AssertRCReturn(rc, rc);
11078
11079	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11080	uint32_t cbOp = 0;
11081	PVM pVM = pVCpu->CTX_SUFF(pVM);
11082	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11083	if (RT_SUCCESS(rc))
11084	{
11085	rc = VINF_SUCCESS;
11086	Assert(cbOp == pDis->cbInstr);
11087	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11088	switch (pDis->pCurInstr->uOpcode)
11089	{
11090	case OP_CLI:
11091	{
11092	pMixedCtx->eflags.Bits.u1IF = 0;
11093	pMixedCtx->rip += pDis->cbInstr;
11094	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11095	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11096	break;
11097	}
11098
11099	case OP_STI:
11100	{
11101	pMixedCtx->eflags.Bits.u1IF = 1;
11102	pMixedCtx->rip += pDis->cbInstr;
11103	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11104	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11105	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11107	break;
11108	}
11109
11110	case OP_HLT:
11111	{
11112	rc = VINF_EM_HALT;
11113	pMixedCtx->rip += pDis->cbInstr;
11114	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11116	break;
11117	}
11118
11119	case OP_POPF:
11120	{
11121	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11122	uint32_t cbParm = 0;
11123	uint32_t uMask = 0;
11124	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11125	{
11126	cbParm = 4;
11127	uMask = 0xffffffff;
11128	}
11129	else
11130	{
11131	cbParm = 2;
11132	uMask = 0xffff;
11133	}
11134
11135	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11136	RTGCPTR GCPtrStack = 0;
11137	X86EFLAGS Eflags;
11138	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11139	&GCPtrStack);
11140	if (RT_SUCCESS(rc))
11141	{
11142	Assert(sizeof(Eflags.u32) >= cbParm);
11143	Eflags.u32 = 0;
11144	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11145	}
11146	if (RT_FAILURE(rc))
11147	{
11148	rc = VERR_EM_INTERPRETER;
11149	break;
11150	}
11151	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11152	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11153	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11154	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11155	pMixedCtx->esp += cbParm;
11156	pMixedCtx->esp &= uMask;
11157	pMixedCtx->rip += pDis->cbInstr;
11158
11159	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11160	\| HM_CHANGED_GUEST_RSP
11161	\| HM_CHANGED_GUEST_RFLAGS);
11162	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11163	break;
11164	}
11165
11166	case OP_PUSHF:
11167	{
11168	uint32_t cbParm = 0;
11169	uint32_t uMask = 0;
11170	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11171	{
11172	cbParm = 4;
11173	uMask = 0xffffffff;
11174	}
11175	else
11176	{
11177	cbParm = 2;
11178	uMask = 0xffff;
11179	}
11180
11181	/* Get the stack pointer & push the contents of eflags onto the stack. */
11182	RTGCPTR GCPtrStack = 0;
11183	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11184	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11185	if (RT_FAILURE(rc))
11186	{
11187	rc = VERR_EM_INTERPRETER;
11188	break;
11189	}
11190	X86EFLAGS Eflags = pMixedCtx->eflags;
11191	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11192	Eflags.Bits.u1RF = 0;
11193	Eflags.Bits.u1VM = 0;
11194
11195	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11196	if (RT_FAILURE(rc))
11197	{
11198	rc = VERR_EM_INTERPRETER;
11199	break;
11200	}
11201	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11202	pMixedCtx->esp -= cbParm;
11203	pMixedCtx->esp &= uMask;
11204	pMixedCtx->rip += pDis->cbInstr;
11205	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11206	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11207	break;
11208	}
11209
11210	case OP_IRET:
11211	{
11212	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11213	* instruction reference. */
11214	RTGCPTR GCPtrStack = 0;
11215	uint32_t uMask = 0xffff;
11216	uint16_t aIretFrame[3];
11217	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11218	{
11219	rc = VERR_EM_INTERPRETER;
11220	break;
11221	}
11222	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11223	&GCPtrStack);
11224	if (RT_SUCCESS(rc))
11225	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11226	if (RT_FAILURE(rc))
11227	{
11228	rc = VERR_EM_INTERPRETER;
11229	break;
11230	}
11231	pMixedCtx->eip = 0;
11232	pMixedCtx->ip = aIretFrame[0];
11233	pMixedCtx->cs.Sel = aIretFrame[1];
11234	pMixedCtx->cs.ValidSel = aIretFrame[1];
11235	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11236	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11237	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11238	pMixedCtx->sp += sizeof(aIretFrame);
11239	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11240	\| HM_CHANGED_GUEST_SEGMENT_REGS
11241	\| HM_CHANGED_GUEST_RSP
11242	\| HM_CHANGED_GUEST_RFLAGS);
11243	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11244	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11245	break;
11246	}
11247
11248	case OP_INT:
11249	{
11250	uint16_t uVector = pDis->Param1.uValue & 0xff;
11251	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11252	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11253	break;
11254	}
11255
11256	case OP_INTO:
11257	{
11258	if (pMixedCtx->eflags.Bits.u1OF)
11259	{
11260	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11261	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11262	}
11263	break;
11264	}
11265
11266	default:
11267	{
11268	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11269	EMCODETYPE_SUPERVISOR);
11270	rc = VBOXSTRICTRC_VAL(rc2);
11271	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11272	Log4(("#GP rc=%Rrc\n", rc));
11273	break;
11274	}
11275	}
11276	}
11277	else
11278	rc = VERR_EM_INTERPRETER;
11279
11280	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11281	("#GP Unexpected rc=%Rrc\n", rc));
11282	return rc;
11283	}
11284
11285
11286	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11287	/**
11288	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11289	* the exception reported in the VMX transient structure back into the VM.
11290	*
11291	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11292	* up-to-date.
11293	*/
11294	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11295	{
11296	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11297
11298	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11299	hmR0VmxCheckExitDueToEventDelivery(). */
11300	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11301	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11302	AssertRCReturn(rc, rc);
11303	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11304
11305	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11306	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11307	return VINF_SUCCESS;
11308	}
11309	#endif
11310
11311
11312	/**
11313	* VM-exit exception handler for #PF (Page-fault exception).
11314	*/
11315	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11316	{
11317	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11318	PVM pVM = pVCpu->CTX_SUFF(pVM);
11319	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11320	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11321	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11322	AssertRCReturn(rc, rc);
11323
11324	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11325	if (pVM->hm.s.fNestedPaging)
11326	{
11327	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11328	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11329	{
11330	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11331	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11332	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11333	}
11334	else
11335	{
11336	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11337	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11338	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11339	}
11340	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11341	return rc;
11342	}
11343	#else
11344	Assert(!pVM->hm.s.fNestedPaging);
11345	NOREF(pVM);
11346	#endif
11347
11348	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11349	AssertRCReturn(rc, rc);
11350
11351	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11352	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11353
11354	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11355	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11356	(RTGCPTR)pVmxTransient->uExitQualification);
11357
11358	Log4(("#PF: rc=%Rrc\n", rc));
11359	if (rc == VINF_SUCCESS)
11360	{
11361	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11362	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11363	* memory? We don't update the whole state here... */
11364	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11365	\| HM_CHANGED_GUEST_RSP
11366	\| HM_CHANGED_GUEST_RFLAGS
11367	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11368	TRPMResetTrap(pVCpu);
11369	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11370	return rc;
11371	}
11372	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11373	{
11374	if (!pVmxTransient->fVectoringPF)
11375	{
11376	/* It's a guest page fault and needs to be reflected to the guest. */
11377	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11378	TRPMResetTrap(pVCpu);
11379	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11380	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11381	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11382	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11383	}
11384	else
11385	{
11386	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11387	TRPMResetTrap(pVCpu);
11388	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11389	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11390	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11391	}
11392
11393	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11394	return VINF_SUCCESS;
11395	}
11396
11397	TRPMResetTrap(pVCpu);
11398	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11399	return rc;
11400	}
11401
11402	/** @} */
11403

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49699

Download in other formats: